Diff of the two buildlogs: -- --- b1/build.log 2024-01-08 10:29:24.021998594 +0000 +++ b2/build.log 2024-01-08 10:39:21.692129778 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sun Feb 9 04:51:11 -12 2025 -I: pbuilder-time-stamp: 1739119871 +I: Current time: Tue Jan 9 00:29:30 +14 2024 +I: pbuilder-time-stamp: 1704709770 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/bookworm-reproducible-base.tgz] I: copying local configuration @@ -31,49 +31,81 @@ dpkg-source: info: applying t-skip--TestRaft_SnapshotRestore_PeerChange.patch I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/2584380/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/D01_modify_environment starting +debug: Running on ionos1-amd64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +'/bin/sh' -> '/bin/bash' +lrwxrwxrwx 1 root root 9 Jan 8 10:30 /bin/sh -> /bin/bash +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build/reproducible-path' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='amd64' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=16 ' - DISTRIBUTION='bookworm' - HOME='/root' - HOST_ARCH='amd64' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:globskipdots:hostcomplete:interactive_comments:patsub_replacement:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_LOADABLES_PATH=/usr/local/lib/bash:/usr/lib/bash:/opt/local/lib/bash:/usr/pkg/lib/bash:/opt/pkg/lib/bash:. + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="2" [2]="15" [3]="1" [4]="release" [5]="x86_64-pc-linux-gnu") + BASH_VERSION='5.2.15(1)-release' + BUILDDIR=/build/reproducible-path + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=amd64 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all parallel=15 ' + DIRSTACK=() + DISTRIBUTION=bookworm + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=x86_64 + HOST_ARCH=amd64 IFS=' ' - INVOCATION_ID='928459a88b304a5da08cf117b482941b' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='2584380' - PS1='# ' - PS2='> ' + INVOCATION_ID=1cdb63706b6348dd87576a53b76f11d8 + LANG=C + LANGUAGE=et_EE:et + LC_ALL=C + MACHTYPE=x86_64-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=1225125 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.5KHihXX9/pbuilderrc_ZUv2 --distribution bookworm --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.5KHihXX9/b1 --logfile b1/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' - SUDO_GID='110' - SUDO_UID='105' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://85.184.249.68:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/r-b-build.5KHihXX9/pbuilderrc_B0r6 --distribution bookworm --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bookworm-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/r-b-build.5KHihXX9/b2 --logfile b2/build.log golang-github-hashicorp-raft_1.1.2-1.dsc' + SUDO_GID=110 + SUDO_UID=105 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://78.137.99.97:3128 I: uname -a - Linux ionos5-amd64 6.5.0-0.deb12.4-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.5.10-1~bpo12+1 (2023-11-23) x86_64 GNU/Linux + Linux i-capture-the-hostname 6.1.0-17-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.69-1 (2023-12-30) x86_64 GNU/Linux I: ls -l /bin total 5632 -rwxr-xr-x 1 root root 1265648 Apr 23 2023 bash @@ -131,15 +163,15 @@ -rwxr-xr-x 1 root root 52112 Sep 20 2022 readlink -rwxr-xr-x 1 root root 72752 Sep 20 2022 rm -rwxr-xr-x 1 root root 56240 Sep 20 2022 rmdir - -rwxr-xr-x 1 root root 27560 Jul 28 2023 run-parts + -rwxr-xr-x 1 root root 27560 Jul 28 23:46 run-parts -rwxr-xr-x 1 root root 126424 Jan 5 2023 sed - lrwxrwxrwx 1 root root 4 Jan 5 2023 sh -> dash + lrwxrwxrwx 1 root root 9 Jan 8 10:30 sh -> /bin/bash -rwxr-xr-x 1 root root 43888 Sep 20 2022 sleep -rwxr-xr-x 1 root root 85008 Sep 20 2022 stty -rwsr-xr-x 1 root root 72000 Mar 23 2023 su -rwxr-xr-x 1 root root 39824 Sep 20 2022 sync -rwxr-xr-x 1 root root 531984 Apr 6 2023 tar - -rwxr-xr-x 1 root root 14520 Jul 28 2023 tempfile + -rwxr-xr-x 1 root root 14520 Jul 28 23:46 tempfile -rwxr-xr-x 1 root root 109616 Sep 20 2022 touch -rwxr-xr-x 1 root root 35664 Sep 20 2022 true -rwxr-xr-x 1 root root 14568 Mar 23 2023 ulockmgr_server @@ -159,7 +191,7 @@ -rwxr-xr-x 1 root root 2206 Apr 10 2022 zless -rwxr-xr-x 1 root root 1842 Apr 10 2022 zmore -rwxr-xr-x 1 root root 4577 Apr 10 2022 znew -I: user script /srv/workspace/pbuilder/2584380/tmp/hooks/D02_print_environment finished +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -303,7 +335,7 @@ Get: 96 http://deb.debian.org/debian bookworm/main amd64 golang-gopkg-check.v1-dev all 0.0+git20200902.038fdea-1 [32.1 kB] Get: 97 http://deb.debian.org/debian bookworm/main amd64 golang-gopkg-mgo.v2-dev all 2016.08.01-7 [316 kB] Get: 98 http://deb.debian.org/debian bookworm/main amd64 golang-github-hashicorp-go-msgpack-dev all 0.5.5-1 [43.3 kB] -Fetched 118 MB in 2s (61.0 MB/s) +Fetched 118 MB in 12s (10.1 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package sensible-utils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 18148 files and directories currently installed.) @@ -715,7 +747,11 @@ fakeroot is already the newest version (1.31-1.2). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/A99_set_merged_usr starting +Not re-configuring usrmerge for bookworm +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/A99_set_merged_usr finished +hostname: Name or service not known +I: Running cd /build/reproducible-path/golang-github-hashicorp-raft-1.1.2/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../golang-github-hashicorp-raft_1.1.2-1_source.changes dpkg-buildpackage: info: source package golang-github-hashicorp-raft dpkg-buildpackage: info: source version 1.1.2-1 dpkg-buildpackage: info: source distribution unstable @@ -733,25 +769,25 @@ dh_autoreconf -O--buildsystem=golang dh_auto_configure -O--buildsystem=golang dh_auto_build -O--buildsystem=golang - cd obj-x86_64-linux-gnu && go install -trimpath -v -p 16 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-x86_64-linux-gnu && go install -trimpath -v -p 15 github.com/hashicorp/raft github.com/hashicorp/raft/bench internal/goarch -crypto/subtle -math/bits internal/unsafeheader +internal/goexperiment internal/goos internal/race -internal/cpu runtime/internal/syscall -internal/goexperiment -crypto/internal/subtle -container/list unicode/utf8 -sync/atomic -internal/itoa unicode -internal/abi +internal/itoa +crypto/internal/subtle +sync/atomic +container/list +math/bits runtime/internal/atomic +internal/cpu +internal/abi runtime/internal/math +crypto/subtle runtime/internal/sys crypto/internal/boring/sig encoding @@ -765,109 +801,109 @@ internal/reflectlite sync internal/testlog -math/rand internal/singleflight internal/sysinfo +math/rand runtime/cgo errors sort internal/oserror -io -internal/safefilepath path github.com/hashicorp/golang-lru/simplelru +io +internal/safefilepath vendor/golang.org/x/net/dns/dnsmessage strconv crypto/internal/nistec/fiat syscall +hash bytes -strings crypto/internal/randutil -hash -hash/crc64 +strings hash/crc32 -vendor/golang.org/x/text/transform +hash/crc64 crypto crypto/rc4 reflect -bufio -github.com/hashicorp/go-immutable-radix net/http/internal/ascii -html regexp/syntax -internal/syscall/unix -internal/syscall/execenv -time +html +github.com/hashicorp/go-immutable-radix +vendor/golang.org/x/text/transform +bufio regexp +time +internal/syscall/execenv +internal/syscall/unix context internal/poll io/fs -embed -os -internal/fmtsort encoding/binary -encoding/base64 +internal/fmtsort +embed +crypto/internal/edwards25519/field crypto/md5 vendor/golang.org/x/crypto/internal/poly1305 -crypto/internal/edwards25519/field -crypto/cipher -crypto/internal/nistec vendor/golang.org/x/crypto/curve25519/internal/field +encoding/base64 +crypto/cipher golang.org/x/sys/unix +crypto/internal/nistec +os encoding/pem -internal/godebug -io/ioutil -fmt -path/filepath -os/signal crypto/internal/edwards25519 -internal/intern -vendor/golang.org/x/sys/cpu crypto/internal/boring -crypto/des vendor/golang.org/x/crypto/chacha20 -net/netip -crypto/aes -crypto/sha1 +crypto/des crypto/hmac crypto/sha256 +crypto/aes crypto/sha512 +crypto/sha1 vendor/golang.org/x/crypto/hkdf +internal/godebug +os/signal +path/filepath +io/ioutil +fmt +internal/intern +net/netip +vendor/golang.org/x/sys/cpu vendor/golang.org/x/crypto/chacha20poly1305 net +math/big log -encoding/json encoding/hex -net/url vendor/golang.org/x/crypto/curve25519 +net/url mime/quotedprintable -compress/flate +net/http/internal mime +compress/flate vendor/golang.org/x/net/http2/hpack -math/big -net/http/internal +encoding/json +encoding/gob vendor/golang.org/x/text/unicode/norm go/token -encoding/gob +github.com/mattn/go-isatty text/template/parse flag runtime/debug -runtime/trace vendor/golang.org/x/text/unicode/bidi -testing -compress/gzip -github.com/mattn/go-isatty -vendor/golang.org/x/text/secure/bidirule github.com/mattn/go-colorable github.com/fatih/color +runtime/trace +compress/gzip +vendor/golang.org/x/text/secure/bidirule +testing text/template github.com/hashicorp/go-hclog -vendor/golang.org/x/net/idna crypto/rand crypto/internal/boring/bbig crypto/elliptic -encoding/asn1 crypto/dsa +encoding/asn1 +vendor/golang.org/x/net/idna crypto/ed25519 crypto/rsa html/template @@ -882,13 +918,13 @@ crypto/tls net/http/httptrace net/http -net/rpc github.com/armon/go-metrics +net/rpc github.com/hashicorp/go-msgpack/codec github.com/hashicorp/raft github.com/hashicorp/raft/bench dh_auto_test -O--buildsystem=golang - cd obj-x86_64-linux-gnu && go test -vet=off -v -p 16 github.com/hashicorp/raft github.com/hashicorp/raft/bench + cd obj-x86_64-linux-gnu && go test -vet=off -v -p 15 github.com/hashicorp/raft github.com/hashicorp/raft/bench === RUN TestCommitment_setVoters --- PASS: TestCommitment_setVoters (0.00s) === RUN TestCommitment_match_max @@ -930,32 +966,32 @@ === RUN TestFileSnapshotSinkImpl --- PASS: TestFileSnapshotSinkImpl (0.00s) === RUN TestFileSS_CreateSnapshotMissingParentDir - testing.go:153: 2025-02-09T16:52:05.674Z [INFO] creating new snapshot: path=/tmp/raft2124179462/raft1841833666/snapshots/3-10-1739119925674.tmp + testing.go:153: 2024-01-08T10:37:51.853Z [INFO] creating new snapshot: path=/tmp/raft4256912193/raft1767633831/snapshots/3-10-1704710271853.tmp --- PASS: TestFileSS_CreateSnapshotMissingParentDir (0.00s) === RUN TestFileSS_CreateSnapshot - testing.go:153: 2025-02-09T16:52:05.676Z [INFO] creating new snapshot: path=/tmp/raft2278030043/snapshots/3-10-1739119925676.tmp - testing.go:153: 2025-02-09T16:52:05.676Z [WARN] found temporary snapshot: name=3-10-1739119925676.tmp + testing.go:153: 2024-01-08T10:37:51.854Z [INFO] creating new snapshot: path=/tmp/raft2187074017/snapshots/3-10-1704710271854.tmp + testing.go:153: 2024-01-08T10:37:51.854Z [WARN] found temporary snapshot: name=3-10-1704710271854.tmp --- PASS: TestFileSS_CreateSnapshot (0.00s) === RUN TestFileSS_CancelSnapshot - testing.go:153: 2025-02-09T16:52:05.677Z [INFO] creating new snapshot: path=/tmp/raft4100629497/snapshots/3-10-1739119925677.tmp + testing.go:153: 2024-01-08T10:37:51.855Z [INFO] creating new snapshot: path=/tmp/raft2847088997/snapshots/3-10-1704710271855.tmp --- PASS: TestFileSS_CancelSnapshot (0.00s) === RUN TestFileSS_Retention - testing.go:153: 2025-02-09T16:52:05.677Z [INFO] creating new snapshot: path=/tmp/raft2699834459/snapshots/3-10-1739119925677.tmp - testing.go:153: 2025-02-09T16:52:05.677Z [INFO] creating new snapshot: path=/tmp/raft2699834459/snapshots/3-11-1739119925677.tmp - testing.go:153: 2025-02-09T16:52:05.677Z [INFO] creating new snapshot: path=/tmp/raft2699834459/snapshots/3-12-1739119925677.tmp - testing.go:153: 2025-02-09T16:52:05.678Z [INFO] reaping snapshot: path=/tmp/raft2699834459/snapshots/3-10-1739119925677 - testing.go:153: 2025-02-09T16:52:05.678Z [INFO] creating new snapshot: path=/tmp/raft2699834459/snapshots/3-13-1739119925678.tmp - testing.go:153: 2025-02-09T16:52:05.678Z [INFO] reaping snapshot: path=/tmp/raft2699834459/snapshots/3-11-1739119925677 - testing.go:153: 2025-02-09T16:52:05.678Z [INFO] creating new snapshot: path=/tmp/raft2699834459/snapshots/3-14-1739119925678.tmp - testing.go:153: 2025-02-09T16:52:05.679Z [INFO] reaping snapshot: path=/tmp/raft2699834459/snapshots/3-12-1739119925677 ---- PASS: TestFileSS_Retention (0.00s) + testing.go:153: 2024-01-08T10:37:51.855Z [INFO] creating new snapshot: path=/tmp/raft1856832098/snapshots/3-10-1704710271855.tmp + testing.go:153: 2024-01-08T10:37:51.855Z [INFO] creating new snapshot: path=/tmp/raft1856832098/snapshots/3-11-1704710271855.tmp + testing.go:153: 2024-01-08T10:37:51.856Z [INFO] creating new snapshot: path=/tmp/raft1856832098/snapshots/3-12-1704710271856.tmp + testing.go:153: 2024-01-08T10:37:51.856Z [INFO] reaping snapshot: path=/tmp/raft1856832098/snapshots/3-10-1704710271855 + testing.go:153: 2024-01-08T10:37:51.856Z [INFO] creating new snapshot: path=/tmp/raft1856832098/snapshots/3-13-1704710271856.tmp + testing.go:153: 2024-01-08T10:37:51.862Z [INFO] reaping snapshot: path=/tmp/raft1856832098/snapshots/3-11-1704710271855 + testing.go:153: 2024-01-08T10:37:51.862Z [INFO] creating new snapshot: path=/tmp/raft1856832098/snapshots/3-14-1704710271862.tmp + testing.go:153: 2024-01-08T10:37:51.863Z [INFO] reaping snapshot: path=/tmp/raft1856832098/snapshots/3-12-1704710271856 +--- PASS: TestFileSS_Retention (0.01s) === RUN TestFileSS_BadPerm --- PASS: TestFileSS_BadPerm (0.00s) === RUN TestFileSS_MissingParentDir --- PASS: TestFileSS_MissingParentDir (0.00s) === RUN TestFileSS_Ordering - testing.go:153: 2025-02-09T16:52:05.680Z [INFO] creating new snapshot: path=/tmp/raft1835152318/snapshots/5-130350-1739119925680.tmp - testing.go:153: 2025-02-09T16:52:05.680Z [INFO] creating new snapshot: path=/tmp/raft1835152318/snapshots/36-204917-1739119925680.tmp + testing.go:153: 2024-01-08T10:37:51.864Z [INFO] creating new snapshot: path=/tmp/raft3062085856/snapshots/5-130350-1704710271864.tmp + testing.go:153: 2024-01-08T10:37:51.864Z [INFO] creating new snapshot: path=/tmp/raft3062085856/snapshots/36-204917-1704710271864.tmp --- PASS: TestFileSS_Ordering (0.00s) === RUN TestDeferFutureSuccess --- PASS: TestDeferFutureSuccess (0.00s) @@ -980,1124 +1016,1126 @@ === RUN TestLogCache --- PASS: TestLogCache (0.00s) === RUN TestNetworkTransport_CloseStreams ---- PASS: TestNetworkTransport_CloseStreams (0.00s) +--- PASS: TestNetworkTransport_CloseStreams (0.02s) === RUN TestNetworkTransport_StartStop --- PASS: TestNetworkTransport_StartStop (0.00s) === RUN TestNetworkTransport_Heartbeat_FastPath ---- PASS: TestNetworkTransport_Heartbeat_FastPath (0.00s) +--- PASS: TestNetworkTransport_Heartbeat_FastPath (0.01s) === RUN TestNetworkTransport_AppendEntries ---- PASS: TestNetworkTransport_AppendEntries (0.00s) +--- PASS: TestNetworkTransport_AppendEntries (0.01s) === RUN TestNetworkTransport_AppendEntriesPipeline ---- PASS: TestNetworkTransport_AppendEntriesPipeline (0.00s) +--- PASS: TestNetworkTransport_AppendEntriesPipeline (0.02s) === RUN TestNetworkTransport_AppendEntriesPipeline_CloseStreams net_transport_test.go:339: Disabled unreliable test --- SKIP: TestNetworkTransport_AppendEntriesPipeline_CloseStreams (0.00s) === RUN TestNetworkTransport_RequestVote ---- PASS: TestNetworkTransport_RequestVote (0.00s) +--- PASS: TestNetworkTransport_RequestVote (0.01s) === RUN TestNetworkTransport_InstallSnapshot ---- PASS: TestNetworkTransport_InstallSnapshot (0.00s) +--- PASS: TestNetworkTransport_InstallSnapshot (0.01s) === RUN TestNetworkTransport_EncodeDecode --- PASS: TestNetworkTransport_EncodeDecode (0.00s) === RUN TestNetworkTransport_EncodeDecode_AddressProvider --- PASS: TestNetworkTransport_EncodeDecode_AddressProvider (0.00s) === RUN TestNetworkTransport_PooledConn ---- PASS: TestNetworkTransport_PooledConn (0.00s) +--- PASS: TestNetworkTransport_PooledConn (0.01s) === RUN TestNetworkTransport_ListenBackoff - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.702Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:51.969Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.707Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:51.977Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.717Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:51.989Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.738Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:52.013Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.778Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:52.055Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:05.859Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:52.136Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:06.020Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:52.298Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:06.341Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:52.618Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:06.982Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:53.262Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:07.986Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:54.270Z [ERROR] test: failed to accept connection: error="intentional error in test" - net_transport_test.go:718: countingWriter: 2025-02-09T16:52:08.990Z [ERROR] test: failed to accept connection: error="intentional error in test" + net_transport_test.go:718: countingWriter: 2024-01-08T10:37:55.274Z [ERROR] test: failed to accept connection: error="intentional error in test" ---- PASS: TestNetworkTransport_ListenBackoff (4.00s) +--- PASS: TestNetworkTransport_ListenBackoff (4.01s) === RUN TestPeersJSON_BadConfiguration --- PASS: TestPeersJSON_BadConfiguration (0.00s) === RUN TestPeersJSON_ReadPeersJSON --- PASS: TestPeersJSON_ReadPeersJSON (0.00s) === RUN TestPeersJSON_ReadConfigJSON ---- PASS: TestPeersJSON_ReadConfigJSON (0.00s) +--- PASS: TestPeersJSON_ReadConfigJSON (0.01s) === RUN TestRaft_StartStop -testLoggerAdapter verbose: server-264fd0ef-0069-9cdc-025b-8a5a1322e172: 2025-02-09T16:52:09.703Z [INFO] server-264fd0ef-0069-9cdc-025b-8a5a1322e172: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-264fd0ef-0069-9cdc-025b-8a5a1322e172 Address:264fd0ef-0069-9cdc-025b-8a5a1322e172}]" - testing.go:149: server-264fd0ef-0069-9cdc-025b-8a5a1322e172: 2025-02-09T16:52:09.703Z [INFO] server-264fd0ef-0069-9cdc-025b-8a5a1322e172: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-264fd0ef-0069-9cdc-025b-8a5a1322e172 Address:264fd0ef-0069-9cdc-025b-8a5a1322e172}]" +testLoggerAdapter verbose: server-afc95e39-c2f8-b04c-3bbb-18a71963eab1: 2024-01-08T10:37:55.983Z [INFO] server-afc95e39-c2f8-b04c-3bbb-18a71963eab1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afc95e39-c2f8-b04c-3bbb-18a71963eab1 Address:afc95e39-c2f8-b04c-3bbb-18a71963eab1}]" + testing.go:149: server-afc95e39-c2f8-b04c-3bbb-18a71963eab1: 2024-01-08T10:37:55.983Z [INFO] server-afc95e39-c2f8-b04c-3bbb-18a71963eab1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-afc95e39-c2f8-b04c-3bbb-18a71963eab1 Address:afc95e39-c2f8-b04c-3bbb-18a71963eab1}]" --- PASS: TestRaft_StartStop (0.00s) === RUN TestRaft_AfterShutdown -testLoggerAdapter verbose: server-bba6e73c-524c-fb57-7523-2f4923854f31: 2025-02-09T16:52:09.704Z [INFO] server-bba6e73c-524c-fb57-7523-2f4923854f31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bba6e73c-524c-fb57-7523-2f4923854f31 Address:bba6e73c-524c-fb57-7523-2f4923854f31}]" - testing.go:149: server-bba6e73c-524c-fb57-7523-2f4923854f31: 2025-02-09T16:52:09.704Z [INFO] server-bba6e73c-524c-fb57-7523-2f4923854f31: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-bba6e73c-524c-fb57-7523-2f4923854f31 Address:bba6e73c-524c-fb57-7523-2f4923854f31}]" ---- PASS: TestRaft_AfterShutdown (0.00s) +testLoggerAdapter verbose: server-53234915-4d71-e881-0775-3a6b5c7093ab: 2024-01-08T10:37:55.985Z [INFO] server-53234915-4d71-e881-0775-3a6b5c7093ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53234915-4d71-e881-0775-3a6b5c7093ab Address:53234915-4d71-e881-0775-3a6b5c7093ab}]" + testing.go:149: server-53234915-4d71-e881-0775-3a6b5c7093ab: 2024-01-08T10:37:55.985Z [INFO] server-53234915-4d71-e881-0775-3a6b5c7093ab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53234915-4d71-e881-0775-3a6b5c7093ab Address:53234915-4d71-e881-0775-3a6b5c7093ab}]" +--- PASS: TestRaft_AfterShutdown (0.01s) === RUN TestRaft_LiveBootstrap -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.704Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: initial configuration: index=0 servers=[] - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.704Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.704Z [INFO] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: initial configuration: index=0 servers=[] - testing.go:149: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.704Z [INFO] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.704Z [INFO] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: initial configuration: index=0 servers=[] - testing.go:149: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.704Z [INFO] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.704Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering follower state: follower="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Follower]" leader= - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.704Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering follower state: follower="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Follower]" leader= -testLoggerAdapter verbose: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.704Z [INFO] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: entering follower state: follower="Node at 5700b5ea-fb27-fbd9-5ab2-8268ffc31035 [Follower]" leader= - testing.go:149: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.704Z [INFO] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: entering follower state: follower="Node at 5700b5ea-fb27-fbd9-5ab2-8268ffc31035 [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:09.704Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:09.704Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.704Z [INFO] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: entering follower state: follower="Node at f252f6d4-f1c3-238b-e28e-557bd991a2ec [Follower]" leader= - testing.go:149: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.704Z [INFO] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: entering follower state: follower="Node at f252f6d4-f1c3-238b-e28e-557bd991a2ec [Follower]" leader= -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering candidate state: node="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Candidate]" term=2 - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering candidate state: node="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Candidate]" term=2 -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: election won: tally=2 - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: election won: tally=2 -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering leader state: leader="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Leader]" - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: entering leader state: leader="Node at 5f15be0d-9a7e-ffce-01fb-83cbe18012a1 [Leader]" -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: added peer, starting replication: peer=server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: added peer, starting replication: peer=server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: added peer, starting replication: peer=server-f252f6d4-f1c3-238b-e28e-557bd991a2ec - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: added peer, starting replication: peer=server-f252f6d4-f1c3-238b-e28e-557bd991a2ec -testLoggerAdapter verbose: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.756Z [WARN] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: 2025-02-09T16:52:09.756Z [WARN] server-f252f6d4-f1c3-238b-e28e-557bd991a2ec: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.756Z [WARN] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: failed to get previous log: previous-index=1 last-index=0 error="log not found" - testing.go:149: server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: 2025-02-09T16:52:09.756Z [WARN] server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035: failed to get previous log: previous-index=1 last-index=0 error="log not found" -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: appendEntries rejected, sending older logs: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" next=1 - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: appendEntries rejected, sending older logs: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" next=1 -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: appendEntries rejected, sending older logs: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" next=1 - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [WARN] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: appendEntries rejected, sending older logs: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" next=1 -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: pipelining replication: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: pipelining replication: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: pipelining replication: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.756Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: pipelining replication: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:09.861Z [INFO] stable state for Follower reached at 2025-02-09 16:52:09.756555682 +0000 UTC m=+4.086788370 (2 nodes), 51.61854ms from start of poll, 52.122415ms from cluster start. Timeout at 2025-02-09 16:52:09.86170084 +0000 UTC m=+4.191933526, 105.145156ms after stability - testing.go:149: cluster: 2025-02-09T16:52:09.861Z [INFO] stable state for Follower reached at 2025-02-09 16:52:09.756555682 +0000 UTC m=+4.086788370 (2 nodes), 51.61854ms from start of poll, 52.122415ms from cluster start. Timeout at 2025-02-09 16:52:09.86170084 +0000 UTC m=+4.191933526, 105.145156ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:09.861Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:09.861Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:09.967Z [INFO] stable state for Leader reached at 2025-02-09 16:52:09.861901531 +0000 UTC m=+4.192134217 (1 nodes), 1.073µs from start of poll, 157.468262ms from cluster start. Timeout at 2025-02-09 16:52:09.967655742 +0000 UTC m=+4.297888428, 105.754211ms after stability - testing.go:149: cluster: 2025-02-09T16:52:09.967Z [INFO] stable state for Leader reached at 2025-02-09 16:52:09.861901531 +0000 UTC m=+4.192134217 (1 nodes), 1.073µs from start of poll, 157.468262ms from cluster start. Timeout at 2025-02-09 16:52:09.967655742 +0000 UTC m=+4.297888428, 105.754211ms after stability -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.978Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: aborting pipeline replication: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.978Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: aborting pipeline replication: peer="{Voter server-f252f6d4-f1c3-238b-e28e-557bd991a2ec f252f6d4-f1c3-238b-e28e-557bd991a2ec}" -testLoggerAdapter verbose: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.978Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: aborting pipeline replication: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" - testing.go:149: server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: 2025-02-09T16:52:09.978Z [INFO] server-5f15be0d-9a7e-ffce-01fb-83cbe18012a1: aborting pipeline replication: peer="{Voter server-5700b5ea-fb27-fbd9-5ab2-8268ffc31035 5700b5ea-fb27-fbd9-5ab2-8268ffc31035}" ---- PASS: TestRaft_LiveBootstrap (0.28s) +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:55.992Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: initial configuration: index=0 servers=[] + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:55.992Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:55.992Z [INFO] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: initial configuration: index=0 servers=[] + testing.go:149: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:55.992Z [INFO] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:55.993Z [INFO] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: initial configuration: index=0 servers=[] + testing.go:149: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:55.993Z [INFO] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:55.993Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering follower state: follower="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Follower]" leader= + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:55.993Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering follower state: follower="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Follower]" leader= +testLoggerAdapter verbose: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:55.994Z [INFO] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: entering follower state: follower="Node at ce8ed5a8-725c-4397-6bda-bc594fbdbeaa [Follower]" leader= +testLoggerAdapter verbose: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:55.994Z [INFO] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: entering follower state: follower="Node at e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 [Follower]" leader= + testing.go:149: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:55.994Z [INFO] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: entering follower state: follower="Node at ce8ed5a8-725c-4397-6bda-bc594fbdbeaa [Follower]" leader= + testing.go:149: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:55.994Z [INFO] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: entering follower state: follower="Node at e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:55.998Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:37:55.998Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering candidate state: node="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Candidate]" term=2 + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering candidate state: node="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Candidate]" term=2 +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: election won: tally=2 + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.074Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: election won: tally=2 +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering leader state: leader="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Leader]" + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: entering leader state: leader="Node at ee2d09f7-1dd2-dffc-9717-f0d84195b395 [Leader]" +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: added peer, starting replication: peer=server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: added peer, starting replication: peer=server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: added peer, starting replication: peer=server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: added peer, starting replication: peer=server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 +testLoggerAdapter verbose: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:56.075Z [WARN] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: 2024-01-08T10:37:56.075Z [WARN] server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: appendEntries rejected, sending older logs: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" next=1 + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.075Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: appendEntries rejected, sending older logs: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" next=1 +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.076Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: pipelining replication: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.076Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: pipelining replication: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" +testLoggerAdapter verbose: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:56.076Z [WARN] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: failed to get previous log: previous-index=1 last-index=0 error="log not found" + testing.go:149: server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: 2024-01-08T10:37:56.076Z [WARN] server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa: failed to get previous log: previous-index=1 last-index=0 error="log not found" +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.077Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: appendEntries rejected, sending older logs: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" next=1 + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.077Z [WARN] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: appendEntries rejected, sending older logs: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" next=1 +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.077Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: pipelining replication: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.077Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: pipelining replication: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.186Z [INFO] stable state for Follower reached at 2024-01-08 10:37:56.078142024 +0000 UTC m=+4.229936175 (2 nodes), 79.602853ms from start of poll, 85.983975ms from cluster start. Timeout at 2024-01-08 10:37:56.186516728 +0000 UTC m=+4.338310876, 108.374701ms after stability + testing.go:149: cluster: 2024-01-08T10:37:56.186Z [INFO] stable state for Follower reached at 2024-01-08 10:37:56.078142024 +0000 UTC m=+4.229936175 (2 nodes), 79.602853ms from start of poll, 85.983975ms from cluster start. Timeout at 2024-01-08 10:37:56.186516728 +0000 UTC m=+4.338310876, 108.374701ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.186Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:56.186Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.294Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.187110724 +0000 UTC m=+4.338904838 (1 nodes), 2.747µs from start of poll, 194.952638ms from cluster start. Timeout at 2024-01-08 10:37:56.294288044 +0000 UTC m=+4.446082170, 107.177332ms after stability + testing.go:149: cluster: 2024-01-08T10:37:56.294Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.187110724 +0000 UTC m=+4.338904838 (1 nodes), 2.747µs from start of poll, 194.952638ms from cluster start. Timeout at 2024-01-08 10:37:56.294288044 +0000 UTC m=+4.446082170, 107.177332ms after stability +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.310Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: aborting pipeline replication: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.310Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: aborting pipeline replication: peer="{Voter server-e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782 e4dd4a3f-a69e-6d69-abe5-9dec2fc5d782}" +testLoggerAdapter verbose: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.310Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: aborting pipeline replication: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" + testing.go:149: server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: 2024-01-08T10:37:56.310Z [INFO] server-ee2d09f7-1dd2-dffc-9717-f0d84195b395: aborting pipeline replication: peer="{Voter server-ce8ed5a8-725c-4397-6bda-bc594fbdbeaa ce8ed5a8-725c-4397-6bda-bc594fbdbeaa}" +--- PASS: TestRaft_LiveBootstrap (0.32s) === RUN TestRaft_RecoverCluster_NoState -testLoggerAdapter verbose: server-b00b061c-871f-9831-cadc-a8574e2c6ffa: 2025-02-09T16:52:09.979Z [INFO] server-b00b061c-871f-9831-cadc-a8574e2c6ffa: initial configuration: index=0 servers=[] - testing.go:149: server-b00b061c-871f-9831-cadc-a8574e2c6ffa: 2025-02-09T16:52:09.979Z [INFO] server-b00b061c-871f-9831-cadc-a8574e2c6ffa: initial configuration: index=0 servers=[] ---- PASS: TestRaft_RecoverCluster_NoState (0.00s) +testLoggerAdapter verbose: server-64024754-4ad1-8e74-e0ee-02947c9a98cd: 2024-01-08T10:37:56.313Z [INFO] server-64024754-4ad1-8e74-e0ee-02947c9a98cd: initial configuration: index=0 servers=[] + testing.go:149: server-64024754-4ad1-8e74-e0ee-02947c9a98cd: 2024-01-08T10:37:56.313Z [INFO] server-64024754-4ad1-8e74-e0ee-02947c9a98cd: initial configuration: index=0 servers=[] +--- PASS: TestRaft_RecoverCluster_NoState (0.01s) === RUN TestRaft_RecoverCluster raft_test.go:119: Disabled unreliable test --- SKIP: TestRaft_RecoverCluster (0.00s) === RUN TestRaft_HasExistingState -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:09.980Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7}]" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:09.980Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7}]" -testLoggerAdapter verbose: server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: 2025-02-09T16:52:09.980Z [INFO] server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7}]" - testing.go:149: server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: 2025-02-09T16:52:09.980Z [INFO] server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7}]" -testLoggerAdapter verbose: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:09.980Z [INFO] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: initial configuration: index=0 servers=[] - testing.go:149: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:09.980Z [INFO] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:09.980Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:09.980Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:09.980Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering follower state: follower="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Follower]" leader= - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:09.980Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering follower state: follower="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Follower]" leader= -testLoggerAdapter verbose: server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: 2025-02-09T16:52:09.980Z [INFO] server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: entering follower state: follower="Node at 2d2d5767-a1d3-dde9-0c42-75588c9761b7 [Follower]" leader= - testing.go:149: server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: 2025-02-09T16:52:09.980Z [INFO] server-2d2d5767-a1d3-dde9-0c42-75588c9761b7: entering follower state: follower="Node at 2d2d5767-a1d3-dde9-0c42-75588c9761b7 [Follower]" leader= -testLoggerAdapter verbose: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:09.980Z [INFO] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: entering follower state: follower="Node at 7067d9d4-3510-167b-0d2c-083d830f8bdb [Follower]" leader= - testing.go:149: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:09.980Z [INFO] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: entering follower state: follower="Node at 7067d9d4-3510-167b-0d2c-083d830f8bdb [Follower]" leader= -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.034Z [WARN] server-ad1a75c9-d83e-a691-a449-1d30094956fb: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.034Z [WARN] server-ad1a75c9-d83e-a691-a449-1d30094956fb: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.034Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering candidate state: node="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Candidate]" term=2 - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.034Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering candidate state: node="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Candidate]" term=2 -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: election won: tally=2 - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: election won: tally=2 -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering leader state: leader="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Leader]" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: entering leader state: leader="Node at ad1a75c9-d83e-a691-a449-1d30094956fb [Leader]" -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: added peer, starting replication: peer=server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: added peer, starting replication: peer=server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: pipelining replication: peer="{Voter server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 2d2d5767-a1d3-dde9-0c42-75588c9761b7}" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.035Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: pipelining replication: peer="{Voter server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 2d2d5767-a1d3-dde9-0c42-75588c9761b7}" -testLoggerAdapter verbose: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:10.043Z [WARN] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: no known peers, aborting election - testing.go:149: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:10.043Z [WARN] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.140Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.035132514 +0000 UTC m=+4.365365206 (1 nodes), 54.598136ms from start of poll, 55.039444ms from cluster start. Timeout at 2025-02-09 16:52:10.140504997 +0000 UTC m=+4.470737683, 105.372477ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.140Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.035132514 +0000 UTC m=+4.365365206 (1 nodes), 54.598136ms from start of poll, 55.039444ms from cluster start. Timeout at 2025-02-09 16:52:10.140504997 +0000 UTC m=+4.470737683, 105.372477ms after stability -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: updating configuration: command=AddStaging server-id=server-7067d9d4-3510-167b-0d2c-083d830f8bdb server-addr=7067d9d4-3510-167b-0d2c-083d830f8bdb servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7} {Suffrage:Voter ID:server-7067d9d4-3510-167b-0d2c-083d830f8bdb Address:7067d9d4-3510-167b-0d2c-083d830f8bdb}]" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: updating configuration: command=AddStaging server-id=server-7067d9d4-3510-167b-0d2c-083d830f8bdb server-addr=7067d9d4-3510-167b-0d2c-083d830f8bdb servers="[{Suffrage:Voter ID:server-ad1a75c9-d83e-a691-a449-1d30094956fb Address:ad1a75c9-d83e-a691-a449-1d30094956fb} {Suffrage:Voter ID:server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 Address:2d2d5767-a1d3-dde9-0c42-75588c9761b7} {Suffrage:Voter ID:server-7067d9d4-3510-167b-0d2c-083d830f8bdb Address:7067d9d4-3510-167b-0d2c-083d830f8bdb}]" -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: added peer, starting replication: peer=server-7067d9d4-3510-167b-0d2c-083d830f8bdb - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: added peer, starting replication: peer=server-7067d9d4-3510-167b-0d2c-083d830f8bdb -testLoggerAdapter verbose: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:10.140Z [WARN] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-7067d9d4-3510-167b-0d2c-083d830f8bdb: 2025-02-09T16:52:10.140Z [WARN] server-7067d9d4-3510-167b-0d2c-083d830f8bdb: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [WARN] server-ad1a75c9-d83e-a691-a449-1d30094956fb: appendEntries rejected, sending older logs: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" next=1 - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [WARN] server-ad1a75c9-d83e-a691-a449-1d30094956fb: appendEntries rejected, sending older logs: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" next=1 -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: pipelining replication: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.140Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: pipelining replication: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.145Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:10.145Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.251Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.146036085 +0000 UTC m=+4.476268771 (1 nodes), 1.101µs from start of poll, 165.943009ms from cluster start. Timeout at 2025-02-09 16:52:10.251145496 +0000 UTC m=+4.581378192, 105.109421ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.251Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.146036085 +0000 UTC m=+4.476268771 (1 nodes), 1.101µs from start of poll, 165.943009ms from cluster start. Timeout at 2025-02-09 16:52:10.251145496 +0000 UTC m=+4.581378192, 105.109421ms after stability -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.251Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: aborting pipeline replication: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.251Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: aborting pipeline replication: peer="{Voter server-7067d9d4-3510-167b-0d2c-083d830f8bdb 7067d9d4-3510-167b-0d2c-083d830f8bdb}" -testLoggerAdapter verbose: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.251Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: aborting pipeline replication: peer="{Voter server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 2d2d5767-a1d3-dde9-0c42-75588c9761b7}" - testing.go:149: server-ad1a75c9-d83e-a691-a449-1d30094956fb: 2025-02-09T16:52:10.251Z [INFO] server-ad1a75c9-d83e-a691-a449-1d30094956fb: aborting pipeline replication: peer="{Voter server-2d2d5767-a1d3-dde9-0c42-75588c9761b7 2d2d5767-a1d3-dde9-0c42-75588c9761b7}" ---- PASS: TestRaft_HasExistingState (0.27s) +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.319Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee}]" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.319Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee}]" +testLoggerAdapter verbose: server-85ba9891-6b1c-fefe-6043-c3341ded65ee: 2024-01-08T10:37:56.320Z [INFO] server-85ba9891-6b1c-fefe-6043-c3341ded65ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee}]" + testing.go:149: server-85ba9891-6b1c-fefe-6043-c3341ded65ee: 2024-01-08T10:37:56.320Z [INFO] server-85ba9891-6b1c-fefe-6043-c3341ded65ee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee}]" +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.320Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering follower state: follower="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Follower]" leader= + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.320Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering follower state: follower="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Follower]" leader= +testLoggerAdapter verbose: server-85ba9891-6b1c-fefe-6043-c3341ded65ee: 2024-01-08T10:37:56.321Z [INFO] server-85ba9891-6b1c-fefe-6043-c3341ded65ee: entering follower state: follower="Node at 85ba9891-6b1c-fefe-6043-c3341ded65ee [Follower]" leader= +testLoggerAdapter verbose: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.321Z [INFO] server-732bb4ac-de8e-e246-f305-da93d9b517f2: initial configuration: index=0 servers=[] + testing.go:149: server-85ba9891-6b1c-fefe-6043-c3341ded65ee: 2024-01-08T10:37:56.321Z [INFO] server-85ba9891-6b1c-fefe-6043-c3341ded65ee: entering follower state: follower="Node at 85ba9891-6b1c-fefe-6043-c3341ded65ee [Follower]" leader= + testing.go:149: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.321Z [INFO] server-732bb4ac-de8e-e246-f305-da93d9b517f2: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.321Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:56.321Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.329Z [INFO] server-732bb4ac-de8e-e246-f305-da93d9b517f2: entering follower state: follower="Node at 732bb4ac-de8e-e246-f305-da93d9b517f2 [Follower]" leader= + testing.go:149: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.329Z [INFO] server-732bb4ac-de8e-e246-f305-da93d9b517f2: entering follower state: follower="Node at 732bb4ac-de8e-e246-f305-da93d9b517f2 [Follower]" leader= +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.414Z [WARN] server-52ec5894-fbca-fa47-0487-be8f97dfa858: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.414Z [WARN] server-52ec5894-fbca-fa47-0487-be8f97dfa858: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.414Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering candidate state: node="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Candidate]" term=2 + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.414Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering candidate state: node="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Candidate]" term=2 +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: election won: tally=2 + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: election won: tally=2 +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering leader state: leader="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Leader]" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: entering leader state: leader="Node at 52ec5894-fbca-fa47-0487-be8f97dfa858 [Leader]" +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: added peer, starting replication: peer=server-85ba9891-6b1c-fefe-6043-c3341ded65ee + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: added peer, starting replication: peer=server-85ba9891-6b1c-fefe-6043-c3341ded65ee +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: pipelining replication: peer="{Voter server-85ba9891-6b1c-fefe-6043-c3341ded65ee 85ba9891-6b1c-fefe-6043-c3341ded65ee}" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.415Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: pipelining replication: peer="{Voter server-85ba9891-6b1c-fefe-6043-c3341ded65ee 85ba9891-6b1c-fefe-6043-c3341ded65ee}" +testLoggerAdapter verbose: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.425Z [WARN] server-732bb4ac-de8e-e246-f305-da93d9b517f2: no known peers, aborting election + testing.go:149: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.425Z [WARN] server-732bb4ac-de8e-e246-f305-da93d9b517f2: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.522Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.416165559 +0000 UTC m=+4.567959713 (1 nodes), 94.447145ms from start of poll, 96.756469ms from cluster start. Timeout at 2024-01-08 10:37:56.521882172 +0000 UTC m=+4.673676374, 105.716661ms after stability + testing.go:149: cluster: 2024-01-08T10:37:56.522Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.416165559 +0000 UTC m=+4.567959713 (1 nodes), 94.447145ms from start of poll, 96.756469ms from cluster start. Timeout at 2024-01-08 10:37:56.521882172 +0000 UTC m=+4.673676374, 105.716661ms after stability +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.522Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: updating configuration: command=AddStaging server-id=server-732bb4ac-de8e-e246-f305-da93d9b517f2 server-addr=732bb4ac-de8e-e246-f305-da93d9b517f2 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee} {Suffrage:Voter ID:server-732bb4ac-de8e-e246-f305-da93d9b517f2 Address:732bb4ac-de8e-e246-f305-da93d9b517f2}]" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.522Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: updating configuration: command=AddStaging server-id=server-732bb4ac-de8e-e246-f305-da93d9b517f2 server-addr=732bb4ac-de8e-e246-f305-da93d9b517f2 servers="[{Suffrage:Voter ID:server-52ec5894-fbca-fa47-0487-be8f97dfa858 Address:52ec5894-fbca-fa47-0487-be8f97dfa858} {Suffrage:Voter ID:server-85ba9891-6b1c-fefe-6043-c3341ded65ee Address:85ba9891-6b1c-fefe-6043-c3341ded65ee} {Suffrage:Voter ID:server-732bb4ac-de8e-e246-f305-da93d9b517f2 Address:732bb4ac-de8e-e246-f305-da93d9b517f2}]" +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.523Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: added peer, starting replication: peer=server-732bb4ac-de8e-e246-f305-da93d9b517f2 + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.523Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: added peer, starting replication: peer=server-732bb4ac-de8e-e246-f305-da93d9b517f2 +testLoggerAdapter verbose: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.523Z [WARN] server-732bb4ac-de8e-e246-f305-da93d9b517f2: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-732bb4ac-de8e-e246-f305-da93d9b517f2: 2024-01-08T10:37:56.523Z [WARN] server-732bb4ac-de8e-e246-f305-da93d9b517f2: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.523Z [WARN] server-52ec5894-fbca-fa47-0487-be8f97dfa858: appendEntries rejected, sending older logs: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" next=1 + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.523Z [WARN] server-52ec5894-fbca-fa47-0487-be8f97dfa858: appendEntries rejected, sending older logs: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" next=1 +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.524Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: pipelining replication: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.524Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: pipelining replication: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.524Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:56.524Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.630Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.524905502 +0000 UTC m=+4.676699630 (1 nodes), 2.218µs from start of poll, 205.496386ms from cluster start. Timeout at 2024-01-08 10:37:56.630021434 +0000 UTC m=+4.781815556, 105.115926ms after stability + testing.go:149: cluster: 2024-01-08T10:37:56.630Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.524905502 +0000 UTC m=+4.676699630 (1 nodes), 2.218µs from start of poll, 205.496386ms from cluster start. Timeout at 2024-01-08 10:37:56.630021434 +0000 UTC m=+4.781815556, 105.115926ms after stability +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.630Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: aborting pipeline replication: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.630Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: aborting pipeline replication: peer="{Voter server-732bb4ac-de8e-e246-f305-da93d9b517f2 732bb4ac-de8e-e246-f305-da93d9b517f2}" +testLoggerAdapter verbose: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.630Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: aborting pipeline replication: peer="{Voter server-85ba9891-6b1c-fefe-6043-c3341ded65ee 85ba9891-6b1c-fefe-6043-c3341ded65ee}" + testing.go:149: server-52ec5894-fbca-fa47-0487-be8f97dfa858: 2024-01-08T10:37:56.630Z [INFO] server-52ec5894-fbca-fa47-0487-be8f97dfa858: aborting pipeline replication: peer="{Voter server-85ba9891-6b1c-fefe-6043-c3341ded65ee 85ba9891-6b1c-fefe-6043-c3341ded65ee}" +--- PASS: TestRaft_HasExistingState (0.31s) === RUN TestRaft_SingleNode -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.252Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 Address:1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3}]" - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.252Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 Address:1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3}]" -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.252Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering follower state: follower="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Follower]" leader= - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.252Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering follower state: follower="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Follower]" leader= -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [WARN] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [WARN] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering candidate state: node="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Candidate]" term=2 - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering candidate state: node="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Candidate]" term=2 -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: election won: tally=1 - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: election won: tally=1 -testLoggerAdapter verbose: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering leader state: leader="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Leader]" - testing.go:149: server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: 2025-02-09T16:52:10.304Z [INFO] server-1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3: entering leader state: leader="Node at 1ba8c490-c839-52c0-b31c-c4c5bcb4bcd3 [Leader]" ---- PASS: TestRaft_SingleNode (0.05s) +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.680Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4f8eeb6-c545-2699-5b54-dc7260cda68d Address:c4f8eeb6-c545-2699-5b54-dc7260cda68d}]" + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.680Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c4f8eeb6-c545-2699-5b54-dc7260cda68d Address:c4f8eeb6-c545-2699-5b54-dc7260cda68d}]" +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.684Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering follower state: follower="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Follower]" leader= + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.684Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering follower state: follower="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Follower]" leader= +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.778Z [WARN] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.778Z [WARN] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.778Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering candidate state: node="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Candidate]" term=2 + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.778Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering candidate state: node="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Candidate]" term=2 +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.779Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: election won: tally=1 + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.779Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: election won: tally=1 +testLoggerAdapter verbose: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.779Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering leader state: leader="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Leader]" + testing.go:149: server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: 2024-01-08T10:37:56.779Z [INFO] server-c4f8eeb6-c545-2699-5b54-dc7260cda68d: entering leader state: leader="Node at c4f8eeb6-c545-2699-5b54-dc7260cda68d [Leader]" +--- PASS: TestRaft_SingleNode (0.15s) === RUN TestRaft_TripleNode -testLoggerAdapter verbose: server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: 2025-02-09T16:52:10.305Z [INFO] server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" - testing.go:149: server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: 2025-02-09T16:52:10.305Z [INFO] server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.305Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.305Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" -testLoggerAdapter verbose: server-5446f2a4-b763-8eae-9429-02a8f38af31d: 2025-02-09T16:52:10.305Z [INFO] server-5446f2a4-b763-8eae-9429-02a8f38af31d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" - testing.go:149: server-5446f2a4-b763-8eae-9429-02a8f38af31d: 2025-02-09T16:52:10.305Z [INFO] server-5446f2a4-b763-8eae-9429-02a8f38af31d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-835d1c4d-f59e-1010-3e1a-e4af53f529fd Address:835d1c4d-f59e-1010-3e1a-e4af53f529fd} {Suffrage:Voter ID:server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 Address:b33657ca-a0e4-91d9-cdd8-55e44e0e9f51} {Suffrage:Voter ID:server-5446f2a4-b763-8eae-9429-02a8f38af31d Address:5446f2a4-b763-8eae-9429-02a8f38af31d}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.305Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:10.305Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: 2025-02-09T16:52:10.305Z [INFO] server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: entering follower state: follower="Node at 835d1c4d-f59e-1010-3e1a-e4af53f529fd [Follower]" leader= - testing.go:149: server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: 2025-02-09T16:52:10.305Z [INFO] server-835d1c4d-f59e-1010-3e1a-e4af53f529fd: entering follower state: follower="Node at 835d1c4d-f59e-1010-3e1a-e4af53f529fd [Follower]" leader= -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.305Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering follower state: follower="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Follower]" leader= - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.305Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering follower state: follower="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Follower]" leader= -testLoggerAdapter verbose: server-5446f2a4-b763-8eae-9429-02a8f38af31d: 2025-02-09T16:52:10.305Z [INFO] server-5446f2a4-b763-8eae-9429-02a8f38af31d: entering follower state: follower="Node at 5446f2a4-b763-8eae-9429-02a8f38af31d [Follower]" leader= - testing.go:149: server-5446f2a4-b763-8eae-9429-02a8f38af31d: 2025-02-09T16:52:10.305Z [INFO] server-5446f2a4-b763-8eae-9429-02a8f38af31d: entering follower state: follower="Node at 5446f2a4-b763-8eae-9429-02a8f38af31d [Follower]" leader= -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.390Z [WARN] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.390Z [WARN] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering candidate state: node="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Candidate]" term=2 - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering candidate state: node="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Candidate]" term=2 -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: election won: tally=2 - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: election won: tally=2 -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering leader state: leader="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Leader]" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: entering leader state: leader="Node at b33657ca-a0e4-91d9-cdd8-55e44e0e9f51 [Leader]" -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: added peer, starting replication: peer=server-835d1c4d-f59e-1010-3e1a-e4af53f529fd - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: added peer, starting replication: peer=server-835d1c4d-f59e-1010-3e1a-e4af53f529fd -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: added peer, starting replication: peer=server-5446f2a4-b763-8eae-9429-02a8f38af31d - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: added peer, starting replication: peer=server-5446f2a4-b763-8eae-9429-02a8f38af31d -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: pipelining replication: peer="{Voter server-5446f2a4-b763-8eae-9429-02a8f38af31d 5446f2a4-b763-8eae-9429-02a8f38af31d}" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: pipelining replication: peer="{Voter server-5446f2a4-b763-8eae-9429-02a8f38af31d 5446f2a4-b763-8eae-9429-02a8f38af31d}" -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: pipelining replication: peer="{Voter server-835d1c4d-f59e-1010-3e1a-e4af53f529fd 835d1c4d-f59e-1010-3e1a-e4af53f529fd}" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.391Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: pipelining replication: peer="{Voter server-835d1c4d-f59e-1010-3e1a-e4af53f529fd 835d1c4d-f59e-1010-3e1a-e4af53f529fd}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.496Z [INFO] stable state for Follower reached at 2025-02-09 16:52:10.391549977 +0000 UTC m=+4.721782666 (2 nodes), 85.74989ms from start of poll, 86.17866ms from cluster start. Timeout at 2025-02-09 16:52:10.496857165 +0000 UTC m=+4.827089851, 105.307185ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.496Z [INFO] stable state for Follower reached at 2025-02-09 16:52:10.391549977 +0000 UTC m=+4.721782666 (2 nodes), 85.74989ms from start of poll, 86.17866ms from cluster start. Timeout at 2025-02-09 16:52:10.496857165 +0000 UTC m=+4.827089851, 105.307185ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.496Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:10.496Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.602Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.496975103 +0000 UTC m=+4.827207791 (1 nodes), 907ns from start of poll, 191.603785ms from cluster start. Timeout at 2025-02-09 16:52:10.602368314 +0000 UTC m=+4.932601000, 105.393209ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.602Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.496975103 +0000 UTC m=+4.827207791 (1 nodes), 907ns from start of poll, 191.603785ms from cluster start. Timeout at 2025-02-09 16:52:10.602368314 +0000 UTC m=+4.932601000, 105.393209ms after stability -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.613Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: aborting pipeline replication: peer="{Voter server-5446f2a4-b763-8eae-9429-02a8f38af31d 5446f2a4-b763-8eae-9429-02a8f38af31d}" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.613Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: aborting pipeline replication: peer="{Voter server-5446f2a4-b763-8eae-9429-02a8f38af31d 5446f2a4-b763-8eae-9429-02a8f38af31d}" -testLoggerAdapter verbose: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.613Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: aborting pipeline replication: peer="{Voter server-835d1c4d-f59e-1010-3e1a-e4af53f529fd 835d1c4d-f59e-1010-3e1a-e4af53f529fd}" - testing.go:149: server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: 2025-02-09T16:52:10.613Z [INFO] server-b33657ca-a0e4-91d9-cdd8-55e44e0e9f51: aborting pipeline replication: peer="{Voter server-835d1c4d-f59e-1010-3e1a-e4af53f529fd 835d1c4d-f59e-1010-3e1a-e4af53f529fd}" ---- PASS: TestRaft_TripleNode (0.31s) +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.783Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.783Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" +testLoggerAdapter verbose: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.784Z [INFO] server-8db61a4d-3ca5-56ee-860c-c752b982adac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" + testing.go:149: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.784Z [INFO] server-8db61a4d-3ca5-56ee-860c-c752b982adac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" +testLoggerAdapter verbose: server-445e5caf-3316-85ac-f3f6-9c49471c7373: 2024-01-08T10:37:56.784Z [INFO] server-445e5caf-3316-85ac-f3f6-9c49471c7373: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" + testing.go:149: server-445e5caf-3316-85ac-f3f6-9c49471c7373: 2024-01-08T10:37:56.784Z [INFO] server-445e5caf-3316-85ac-f3f6-9c49471c7373: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ef15228e-5fb1-958d-b7f9-66dedbb7121d Address:ef15228e-5fb1-958d-b7f9-66dedbb7121d} {Suffrage:Voter ID:server-8db61a4d-3ca5-56ee-860c-c752b982adac Address:8db61a4d-3ca5-56ee-860c-c752b982adac} {Suffrage:Voter ID:server-445e5caf-3316-85ac-f3f6-9c49471c7373 Address:445e5caf-3316-85ac-f3f6-9c49471c7373}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.785Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:37:56.785Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.785Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering follower state: follower="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Follower]" leader= + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.785Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering follower state: follower="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Follower]" leader= +testLoggerAdapter verbose: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.785Z [INFO] server-8db61a4d-3ca5-56ee-860c-c752b982adac: entering follower state: follower="Node at 8db61a4d-3ca5-56ee-860c-c752b982adac [Follower]" leader= + testing.go:149: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.785Z [INFO] server-8db61a4d-3ca5-56ee-860c-c752b982adac: entering follower state: follower="Node at 8db61a4d-3ca5-56ee-860c-c752b982adac [Follower]" leader= +testLoggerAdapter verbose: server-445e5caf-3316-85ac-f3f6-9c49471c7373: 2024-01-08T10:37:56.786Z [INFO] server-445e5caf-3316-85ac-f3f6-9c49471c7373: entering follower state: follower="Node at 445e5caf-3316-85ac-f3f6-9c49471c7373 [Follower]" leader= + testing.go:149: server-445e5caf-3316-85ac-f3f6-9c49471c7373: 2024-01-08T10:37:56.786Z [INFO] server-445e5caf-3316-85ac-f3f6-9c49471c7373: entering follower state: follower="Node at 445e5caf-3316-85ac-f3f6-9c49471c7373 [Follower]" leader= +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.862Z [WARN] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.862Z [WARN] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.862Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering candidate state: node="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Candidate]" term=2 + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.862Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering candidate state: node="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Candidate]" term=2 +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: election won: tally=2 + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: election won: tally=2 +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering leader state: leader="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Leader]" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: entering leader state: leader="Node at ef15228e-5fb1-958d-b7f9-66dedbb7121d [Leader]" +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: added peer, starting replication: peer=server-8db61a4d-3ca5-56ee-860c-c752b982adac + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: added peer, starting replication: peer=server-8db61a4d-3ca5-56ee-860c-c752b982adac +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: added peer, starting replication: peer=server-445e5caf-3316-85ac-f3f6-9c49471c7373 + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: added peer, starting replication: peer=server-445e5caf-3316-85ac-f3f6-9c49471c7373 +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: pipelining replication: peer="{Voter server-445e5caf-3316-85ac-f3f6-9c49471c7373 445e5caf-3316-85ac-f3f6-9c49471c7373}" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.863Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: pipelining replication: peer="{Voter server-445e5caf-3316-85ac-f3f6-9c49471c7373 445e5caf-3316-85ac-f3f6-9c49471c7373}" +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.864Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: pipelining replication: peer="{Voter server-8db61a4d-3ca5-56ee-860c-c752b982adac 8db61a4d-3ca5-56ee-860c-c752b982adac}" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:56.864Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: pipelining replication: peer="{Voter server-8db61a4d-3ca5-56ee-860c-c752b982adac 8db61a4d-3ca5-56ee-860c-c752b982adac}" +testLoggerAdapter verbose: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.864Z [WARN] server-8db61a4d-3ca5-56ee-860c-c752b982adac: rejecting vote request since our last term is greater: candidate=ef15228e-5fb1-958d-b7f9-66dedbb7121d last-term=2 last-candidate-term=1 + testing.go:149: server-8db61a4d-3ca5-56ee-860c-c752b982adac: 2024-01-08T10:37:56.864Z [WARN] server-8db61a4d-3ca5-56ee-860c-c752b982adac: rejecting vote request since our last term is greater: candidate=ef15228e-5fb1-958d-b7f9-66dedbb7121d last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.980Z [INFO] stable state for Follower reached at 2024-01-08 10:37:56.86515134 +0000 UTC m=+5.016945490 (2 nodes), 79.731262ms from start of poll, 83.505891ms from cluster start. Timeout at 2024-01-08 10:37:56.980222053 +0000 UTC m=+5.132016343, 115.070853ms after stability + testing.go:149: cluster: 2024-01-08T10:37:56.980Z [INFO] stable state for Follower reached at 2024-01-08 10:37:56.86515134 +0000 UTC m=+5.016945490 (2 nodes), 79.731262ms from start of poll, 83.505891ms from cluster start. Timeout at 2024-01-08 10:37:56.980222053 +0000 UTC m=+5.132016343, 115.070853ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:56.980Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:56.980Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.086Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.980922148 +0000 UTC m=+5.132716268 (1 nodes), 3.322µs from start of poll, 199.276669ms from cluster start. Timeout at 2024-01-08 10:37:57.086417141 +0000 UTC m=+5.238211261, 105.494993ms after stability + testing.go:149: cluster: 2024-01-08T10:37:57.086Z [INFO] stable state for Leader reached at 2024-01-08 10:37:56.980922148 +0000 UTC m=+5.132716268 (1 nodes), 3.322µs from start of poll, 199.276669ms from cluster start. Timeout at 2024-01-08 10:37:57.086417141 +0000 UTC m=+5.238211261, 105.494993ms after stability +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:57.103Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: aborting pipeline replication: peer="{Voter server-445e5caf-3316-85ac-f3f6-9c49471c7373 445e5caf-3316-85ac-f3f6-9c49471c7373}" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:57.103Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: aborting pipeline replication: peer="{Voter server-445e5caf-3316-85ac-f3f6-9c49471c7373 445e5caf-3316-85ac-f3f6-9c49471c7373}" +testLoggerAdapter verbose: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:57.103Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: aborting pipeline replication: peer="{Voter server-8db61a4d-3ca5-56ee-860c-c752b982adac 8db61a4d-3ca5-56ee-860c-c752b982adac}" + testing.go:149: server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: 2024-01-08T10:37:57.103Z [INFO] server-ef15228e-5fb1-958d-b7f9-66dedbb7121d: aborting pipeline replication: peer="{Voter server-8db61a4d-3ca5-56ee-860c-c752b982adac 8db61a4d-3ca5-56ee-860c-c752b982adac}" +--- PASS: TestRaft_TripleNode (0.33s) === RUN TestRaft_LeaderFail -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.614Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.614Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.614Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering follower state: follower="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Follower]" leader= -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.614Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.614Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering follower state: follower="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Follower]" leader= - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.614Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.614Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.614Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ed3df739-60f3-04f4-cdd3-6aa793b08087 Address:ed3df739-60f3-04f4-cdd3-6aa793b08087} {Suffrage:Voter ID:server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a Address:c7bfc503-0499-6397-f90c-b7fe4bb4db0a} {Suffrage:Voter ID:server-77661ac0-09bb-b789-ba94-ff98cbd2fafc Address:77661ac0-09bb-b789-ba94-ff98cbd2fafc}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.614Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:10.614Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.614Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader= - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.614Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader= -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.614Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering follower state: follower="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Follower]" leader= - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.614Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering follower state: follower="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Follower]" leader= -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.675Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.675Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.675Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=2 - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.675Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=2 -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: election won: tally=2 - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: election won: tally=2 -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering leader state: leader="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Leader]" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering leader state: leader="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Leader]" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: added peer, starting replication: peer=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: added peer, starting replication: peer=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: added peer, starting replication: peer=server-77661ac0-09bb-b789-ba94-ff98cbd2fafc - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: added peer, starting replication: peer=server-77661ac0-09bb-b789-ba94-ff98cbd2fafc -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: pipelining replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: pipelining replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: pipelining replication: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.676Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: pipelining replication: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.781Z [INFO] stable state for Follower reached at 2025-02-09 16:52:10.676130966 +0000 UTC m=+5.006363654 (2 nodes), 61.250354ms from start of poll, 61.780636ms from cluster start. Timeout at 2025-02-09 16:52:10.78131815 +0000 UTC m=+5.111550838, 105.187184ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.781Z [INFO] stable state for Follower reached at 2025-02-09 16:52:10.676130966 +0000 UTC m=+5.006363654 (2 nodes), 61.250354ms from start of poll, 61.780636ms from cluster start. Timeout at 2025-02-09 16:52:10.78131815 +0000 UTC m=+5.111550838, 105.187184ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.781Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:10.781Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.886Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.781447474 +0000 UTC m=+5.111680160 (1 nodes), 914ns from start of poll, 167.097142ms from cluster start. Timeout at 2025-02-09 16:52:10.886457094 +0000 UTC m=+5.216689780, 105.00962ms after stability - testing.go:149: cluster: 2025-02-09T16:52:10.886Z [INFO] stable state for Leader reached at 2025-02-09 16:52:10.781447474 +0000 UTC m=+5.111680160 (1 nodes), 914ns from start of poll, 167.097142ms from cluster start. Timeout at 2025-02-09 16:52:10.886457094 +0000 UTC m=+5.216689780, 105.00962ms after stability - raft_test.go:336: [INFO] Disconnecting Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Leader] -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.898Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.898Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.900Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to pipeline appendEntries: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="append pipeline closed" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.900Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to pipeline appendEntries: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="append pipeline closed" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.900Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: aborting pipeline replication: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.900Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: aborting pipeline replication: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:10.903Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:10.903Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.905Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to pipeline appendEntries: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="append pipeline closed" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.905Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to pipeline appendEntries: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="append pipeline closed" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.905Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: aborting pipeline replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.905Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: aborting pipeline replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.907Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.907Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.910Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.910Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.911Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.911Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.917Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.917Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.928Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.928Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.928Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.928Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.932Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.932Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.935Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.935Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.944Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.944Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.945Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 time=50.400081ms - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.945Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 time=50.400081ms -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.948Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.948Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=77661ac0-09bb-b789-ba94-ff98cbd2fafc error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.950Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.950Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 time=61.271419ms - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 time=61.271419ms -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-77661ac0-09bb-b789-ba94-ff98cbd2fafc time=59.075846ms - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact: server-id=server-77661ac0-09bb-b789-ba94-ff98cbd2fafc time=59.075846ms -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact quorum of nodes, stepping down - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader= - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.956Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader= -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.960Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: heartbeat timeout reached, starting election: last-leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.960Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: heartbeat timeout reached, starting election: last-leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.961Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering candidate state: node="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Candidate]" term=3 - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.961Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering candidate state: node="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Candidate]" term=3 -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.961Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:10.961Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.961Z [WARN] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: rejecting vote request since we have a leader: from=77661ac0-09bb-b789-ba94-ff98cbd2fafc leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:10.961Z [WARN] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: rejecting vote request since we have a leader: from=77661ac0-09bb-b789-ba94-ff98cbd2fafc leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.967Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.967Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to heartbeat to: peer=ed3df739-60f3-04f4-cdd3-6aa793b08087 error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.970Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.970Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.977Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:10.977Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to appendEntries to: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [WARN] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: heartbeat timeout reached, starting election: last-leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [WARN] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: heartbeat timeout reached, starting election: last-leader=c7bfc503-0499-6397-f90c-b7fe4bb4db0a -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering candidate state: node="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Candidate]" term=3 - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering candidate state: node="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Candidate]" term=3 -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.000Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: duplicate requestVote for same term: term=3 - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.000Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [ERROR] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.000Z [ERROR] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.042Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.042Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=3 - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=3 -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.043Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: Election timeout reached, restarting election - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: Election timeout reached, restarting election -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering candidate state: node="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Candidate]" term=4 - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering candidate state: node="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Candidate]" term=4 -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to make requestVote RPC: target="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.048Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering follower state: follower="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Follower]" leader= - testing.go:149: server-ed3df739-60f3-04f4-cdd3-6aa793b08087: 2025-02-09T16:52:11.048Z [INFO] server-ed3df739-60f3-04f4-cdd3-6aa793b08087: entering follower state: follower="Node at ed3df739-60f3-04f4-cdd3-6aa793b08087 [Follower]" leader= -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: election won: tally=2 - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: election won: tally=2 -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering leader state: leader="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Leader]" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: entering leader state: leader="Node at 77661ac0-09bb-b789-ba94-ff98cbd2fafc [Leader]" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: added peer, starting replication: peer=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: added peer, starting replication: peer=server-ed3df739-60f3-04f4-cdd3-6aa793b08087 -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: added peer, starting replication: peer=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: added peer, starting replication: peer=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: pipelining replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.048Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: pipelining replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.055Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.055Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.069Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.069Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.074Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.074Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.089Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.089Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.094Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.094Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.099Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to contact: server-id=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a time=50.559611ms - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.099Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to contact: server-id=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a time=50.559611ms -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.119Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.119Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to appendEntries to: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.122Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.122Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="failed to connect to peer: c7bfc503-0499-6397-f90c-b7fe4bb4db0a" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: Election timeout reached, restarting election - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [WARN] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: Election timeout reached, restarting election -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=4 - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering candidate state: node="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate]" term=4 -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-77661ac0-09bb-b789-ba94-ff98cbd2fafc 77661ac0-09bb-b789-ba94-ff98cbd2fafc}" error="failed to connect to peer: 77661ac0-09bb-b789-ba94-ff98cbd2fafc" -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.137Z [ERROR] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: failed to make requestVote RPC: target="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" error="failed to connect to peer: ed3df739-60f3-04f4-cdd3-6aa793b08087" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.147Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to contact: server-id=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a time=98.273173ms - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.147Z [WARN] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to contact: server-id=server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a time=98.273173ms -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.154Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.04890515 +0000 UTC m=+5.379137838 (1 nodes), 145.443908ms from start of poll, 434.55482ms from cluster start. Timeout at 2025-02-09 16:52:11.154714936 +0000 UTC m=+5.484947625, 105.809787ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.154Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.04890515 +0000 UTC m=+5.379137838 (1 nodes), 145.443908ms from start of poll, 434.55482ms from cluster start. Timeout at 2025-02-09 16:52:11.154714936 +0000 UTC m=+5.484947625, 105.809787ms after stability - raft_test.go:371: [INFO] Reconnecting Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Candidate] -testLoggerAdapter verbose: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.165Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader=77661ac0-09bb-b789-ba94-ff98cbd2fafc - testing.go:149: server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: 2025-02-09T16:52:11.165Z [INFO] server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a: entering follower state: follower="Node at c7bfc503-0499-6397-f90c-b7fe4bb4db0a [Follower]" leader=77661ac0-09bb-b789-ba94-ff98cbd2fafc -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.166Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: pipelining replication: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.166Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: pipelining replication: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.171Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: aborting pipeline replication: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.171Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: aborting pipeline replication: peer="{Voter server-c7bfc503-0499-6397-f90c-b7fe4bb4db0a c7bfc503-0499-6397-f90c-b7fe4bb4db0a}" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.171Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: aborting pipeline replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.171Z [INFO] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: aborting pipeline replication: peer="{Voter server-ed3df739-60f3-04f4-cdd3-6aa793b08087 ed3df739-60f3-04f4-cdd3-6aa793b08087}" -testLoggerAdapter verbose: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.221Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="command timed out" - testing.go:149: server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: 2025-02-09T16:52:11.221Z [ERROR] server-77661ac0-09bb-b789-ba94-ff98cbd2fafc: failed to heartbeat to: peer=c7bfc503-0499-6397-f90c-b7fe4bb4db0a error="command timed out" ---- PASS: TestRaft_LeaderFail (0.61s) +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.122Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.122Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.123Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.123Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.123Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.123Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 Address:ccc441d5-5ed3-b0ff-1334-1c1f12e83de1} {Suffrage:Voter ID:server-294f74a8-d626-23aa-d7df-0123d027628c Address:294f74a8-d626-23aa-d7df-0123d027628c} {Suffrage:Voter ID:server-fff0e3e6-f861-221f-5da4-b30db566d071 Address:fff0e3e6-f861-221f-5da4-b30db566d071}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.123Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.123Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T10:37:57.123Z [INFO] starting stability test: raft-state=Follower + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.123Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader= +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.124Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering follower state: follower="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Follower]" leader= + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.124Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering follower state: follower="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Follower]" leader= +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.124Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering follower state: follower="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Follower]" leader= + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.124Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering follower state: follower="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Follower]" leader= +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=2 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=2 +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: election won: tally=2 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: election won: tally=2 +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering leader state: leader="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Leader]" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering leader state: leader="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Leader]" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: added peer, starting replication: peer=server-294f74a8-d626-23aa-d7df-0123d027628c + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: added peer, starting replication: peer=server-294f74a8-d626-23aa-d7df-0123d027628c +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: added peer, starting replication: peer=server-fff0e3e6-f861-221f-5da4-b30db566d071 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: added peer, starting replication: peer=server-fff0e3e6-f861-221f-5da4-b30db566d071 +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: pipelining replication: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: pipelining replication: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: pipelining replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.199Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: pipelining replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.306Z [INFO] stable state for Follower reached at 2024-01-08 10:37:57.199874863 +0000 UTC m=+5.351668714 (2 nodes), 75.837938ms from start of poll, 78.532471ms from cluster start. Timeout at 2024-01-08 10:37:57.306042094 +0000 UTC m=+5.457836216, 106.167502ms after stability + testing.go:149: cluster: 2024-01-08T10:37:57.306Z [INFO] stable state for Follower reached at 2024-01-08 10:37:57.199874863 +0000 UTC m=+5.351668714 (2 nodes), 75.837938ms from start of poll, 78.532471ms from cluster start. Timeout at 2024-01-08 10:37:57.306042094 +0000 UTC m=+5.457836216, 106.167502ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.306Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:57.306Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.414Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.306776009 +0000 UTC m=+5.458570127 (1 nodes), 2.854µs from start of poll, 185.433884ms from cluster start. Timeout at 2024-01-08 10:37:57.413910525 +0000 UTC m=+5.565704653, 107.134526ms after stability + testing.go:149: cluster: 2024-01-08T10:37:57.414Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.306776009 +0000 UTC m=+5.458570127 (1 nodes), 2.854µs from start of poll, 185.433884ms from cluster start. Timeout at 2024-01-08 10:37:57.413910525 +0000 UTC m=+5.565704653, 107.134526ms after stability + raft_test.go:336: [INFO] Disconnecting Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Leader] +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.426Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.426Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.427Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.427Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.433Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:57.433Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.433Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to pipeline appendEntries: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="append pipeline closed" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.433Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to pipeline appendEntries: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="append pipeline closed" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.433Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: aborting pipeline replication: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.433Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: aborting pipeline replication: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.435Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to pipeline appendEntries: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="append pipeline closed" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.435Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to pipeline appendEntries: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="append pipeline closed" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.435Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: aborting pipeline replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.435Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: aborting pipeline replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.442Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.442Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.443Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.443Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.444Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.444Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.447Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.447Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.460Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.460Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=294f74a8-d626-23aa-d7df-0123d027628c error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.462Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.462Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.464Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.464Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.465Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.465Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to heartbeat to: peer=fff0e3e6-f861-221f-5da4-b30db566d071 error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.473Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-fff0e3e6-f861-221f-5da4-b30db566d071 time=52.596144ms + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.473Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-fff0e3e6-f861-221f-5da4-b30db566d071 time=52.596144ms +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-294f74a8-d626-23aa-d7df-0123d027628c time=61.692072ms + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-294f74a8-d626-23aa-d7df-0123d027628c time=61.692072ms +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-fff0e3e6-f861-221f-5da4-b30db566d071 time=65.279417ms + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact: server-id=server-fff0e3e6-f861-221f-5da4-b30db566d071 time=65.279417ms +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact quorum of nodes, stepping down + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader= + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.486Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader= +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.487Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.487Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to appendEntries to: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.522Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: heartbeat timeout reached, starting election: last-leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.522Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: heartbeat timeout reached, starting election: last-leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.522Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering candidate state: node="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Candidate]" term=3 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.522Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering candidate state: node="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Candidate]" term=3 +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.522Z [WARN] server-294f74a8-d626-23aa-d7df-0123d027628c: rejecting vote request since we have a leader: from=fff0e3e6-f861-221f-5da4-b30db566d071 leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.522Z [WARN] server-294f74a8-d626-23aa-d7df-0123d027628c: rejecting vote request since we have a leader: from=fff0e3e6-f861-221f-5da4-b30db566d071 leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.523Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.523Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.546Z [WARN] server-294f74a8-d626-23aa-d7df-0123d027628c: heartbeat timeout reached, starting election: last-leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.546Z [WARN] server-294f74a8-d626-23aa-d7df-0123d027628c: heartbeat timeout reached, starting election: last-leader=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.546Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering candidate state: node="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Candidate]" term=3 + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.546Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering candidate state: node="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Candidate]" term=3 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.546Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: duplicate requestVote for same term: term=3 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.546Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.547Z [ERROR] server-294f74a8-d626-23aa-d7df-0123d027628c: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.547Z [ERROR] server-294f74a8-d626-23aa-d7df-0123d027628c: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=3 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=3 +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.582Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.601Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: Election timeout reached, restarting election + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.601Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: Election timeout reached, restarting election +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.602Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering candidate state: node="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Candidate]" term=4 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.602Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering candidate state: node="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Candidate]" term=4 +testLoggerAdapter verbose: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.602Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering follower state: follower="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Follower]" leader= + testing.go:149: server-294f74a8-d626-23aa-d7df-0123d027628c: 2024-01-08T10:37:57.602Z [INFO] server-294f74a8-d626-23aa-d7df-0123d027628c: entering follower state: follower="Node at 294f74a8-d626-23aa-d7df-0123d027628c [Follower]" leader= +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.602Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: election won: tally=2 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.602Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: election won: tally=2 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering leader state: leader="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Leader]" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: entering leader state: leader="Node at fff0e3e6-f861-221f-5da4-b30db566d071 [Leader]" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: added peer, starting replication: peer=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: added peer, starting replication: peer=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: added peer, starting replication: peer=server-294f74a8-d626-23aa-d7df-0123d027628c + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: added peer, starting replication: peer=server-294f74a8-d626-23aa-d7df-0123d027628c +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: pipelining replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.603Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: pipelining replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.604Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.604Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to make requestVote RPC: target="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.604Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.604Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.611Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.611Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.638Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.638Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.638Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.638Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.653Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to contact: server-id=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 time=50.436127ms + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.653Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to contact: server-id=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 time=50.436127ms +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.657Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.657Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.662Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.662Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.677Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: Election timeout reached, restarting election + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.677Z [WARN] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: Election timeout reached, restarting election +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.678Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=4 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.678Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering candidate state: node="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate]" term=4 +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.678Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.678Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-fff0e3e6-f861-221f-5da4-b30db566d071 fff0e3e6-f861-221f-5da4-b30db566d071}" error="failed to connect to peer: fff0e3e6-f861-221f-5da4-b30db566d071" +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.679Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.679Z [ERROR] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: failed to make requestVote RPC: target="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" error="failed to connect to peer: 294f74a8-d626-23aa-d7df-0123d027628c" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.690Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.690Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to heartbeat to: peer=ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.698Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.698Z [ERROR] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to appendEntries to: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" error="failed to connect to peer: ccc441d5-5ed3-b0ff-1334-1c1f12e83de1" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.701Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to contact: server-id=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 time=98.457441ms + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.701Z [WARN] server-fff0e3e6-f861-221f-5da4-b30db566d071: failed to contact: server-id=server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 time=98.457441ms +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.710Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.604638139 +0000 UTC m=+5.756432287 (1 nodes), 171.255747ms from start of poll, 483.296044ms from cluster start. Timeout at 2024-01-08 10:37:57.709923094 +0000 UTC m=+5.861717231, 105.284944ms after stability + testing.go:149: cluster: 2024-01-08T10:37:57.710Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.604638139 +0000 UTC m=+5.756432287 (1 nodes), 171.255747ms from start of poll, 483.296044ms from cluster start. Timeout at 2024-01-08 10:37:57.709923094 +0000 UTC m=+5.861717231, 105.284944ms after stability + raft_test.go:371: [INFO] Reconnecting Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Candidate] +testLoggerAdapter verbose: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.746Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader=fff0e3e6-f861-221f-5da4-b30db566d071 + testing.go:149: server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: 2024-01-08T10:37:57.746Z [INFO] server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1: entering follower state: follower="Node at ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 [Follower]" leader=fff0e3e6-f861-221f-5da4-b30db566d071 +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.751Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: pipelining replication: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.751Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: pipelining replication: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.754Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: aborting pipeline replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.754Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: aborting pipeline replication: peer="{Voter server-294f74a8-d626-23aa-d7df-0123d027628c 294f74a8-d626-23aa-d7df-0123d027628c}" +testLoggerAdapter verbose: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.754Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: aborting pipeline replication: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" + testing.go:149: server-fff0e3e6-f861-221f-5da4-b30db566d071: 2024-01-08T10:37:57.754Z [INFO] server-fff0e3e6-f861-221f-5da4-b30db566d071: aborting pipeline replication: peer="{Voter server-ccc441d5-5ed3-b0ff-1334-1c1f12e83de1 ccc441d5-5ed3-b0ff-1334-1c1f12e83de1}" +--- PASS: TestRaft_LeaderFail (0.65s) === RUN TestRaft_BehindFollower -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.222Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.222Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" -testLoggerAdapter verbose: server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: 2025-02-09T16:52:11.222Z [INFO] server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" - testing.go:149: server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: 2025-02-09T16:52:11.222Z [INFO] server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" -testLoggerAdapter verbose: server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: 2025-02-09T16:52:11.223Z [INFO] server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" - testing.go:149: server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: 2025-02-09T16:52:11.223Z [INFO] server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06ffd059-69aa-e354-e84c-f264a1c5f4d8 Address:06ffd059-69aa-e354-e84c-f264a1c5f4d8} {Suffrage:Voter ID:server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 Address:0936aadc-eb10-aaff-a82c-3a2bce0bf5d6} {Suffrage:Voter ID:server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 Address:77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.223Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:11.223Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.223Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering follower state: follower="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Follower]" leader= - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.223Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering follower state: follower="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Follower]" leader= -testLoggerAdapter verbose: server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: 2025-02-09T16:52:11.223Z [INFO] server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: entering follower state: follower="Node at 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 [Follower]" leader= - testing.go:149: server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: 2025-02-09T16:52:11.223Z [INFO] server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6: entering follower state: follower="Node at 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 [Follower]" leader= -testLoggerAdapter verbose: server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: 2025-02-09T16:52:11.223Z [INFO] server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: entering follower state: follower="Node at 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 [Follower]" leader= - testing.go:149: server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: 2025-02-09T16:52:11.223Z [INFO] server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795: entering follower state: follower="Node at 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 [Follower]" leader= -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [WARN] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [WARN] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering candidate state: node="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Candidate]" term=2 - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering candidate state: node="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Candidate]" term=2 -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: election won: tally=2 - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: election won: tally=2 -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering leader state: leader="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Leader]" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: entering leader state: leader="Node at 06ffd059-69aa-e354-e84c-f264a1c5f4d8 [Leader]" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: added peer, starting replication: peer=server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: added peer, starting replication: peer=server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: added peer, starting replication: peer=server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: added peer, starting replication: peer=server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.290Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.396Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.29101062 +0000 UTC m=+5.621243316 (1 nodes), 67.911602ms from start of poll, 68.344374ms from cluster start. Timeout at 2025-02-09 16:52:11.396634524 +0000 UTC m=+5.726867210, 105.623894ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.396Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.29101062 +0000 UTC m=+5.621243316 (1 nodes), 67.911602ms from start of poll, 68.344374ms from cluster start. Timeout at 2025-02-09 16:52:11.396634524 +0000 UTC m=+5.726867210, 105.623894ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.396Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:11.396Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.502Z [INFO] stable state for Follower reached at 2025-02-09 16:52:11.3967714 +0000 UTC m=+5.727004088 (2 nodes), 970ns from start of poll, 174.105146ms from cluster start. Timeout at 2025-02-09 16:52:11.502442277 +0000 UTC m=+5.832674965, 105.670877ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.502Z [INFO] stable state for Follower reached at 2025-02-09 16:52:11.3967714 +0000 UTC m=+5.727004088 (2 nodes), 970ns from start of poll, 174.105146ms from cluster start. Timeout at 2025-02-09 16:52:11.502442277 +0000 UTC m=+5.832674965, 105.670877ms after stability -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to pipeline appendEntries: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" error="append pipeline closed" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to pipeline appendEntries: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" error="append pipeline closed" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to appendEntries to: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" error="failed to connect to peer: 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.502Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to appendEntries to: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" error="failed to connect to peer: 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.503Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to heartbeat to: peer=0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 error="failed to connect to peer: 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.503Z [ERROR] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: failed to heartbeat to: peer=0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 error="failed to connect to peer: 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6" +testLoggerAdapter verbose: server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: 2024-01-08T10:37:57.769Z [INFO] server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" + testing.go:149: server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: 2024-01-08T10:37:57.769Z [INFO] server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.770Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.770Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" +testLoggerAdapter verbose: server-554072fc-a8a1-bf41-8ac6-8596225fc000: 2024-01-08T10:37:57.771Z [INFO] server-554072fc-a8a1-bf41-8ac6-8596225fc000: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" + testing.go:149: server-554072fc-a8a1-bf41-8ac6-8596225fc000: 2024-01-08T10:37:57.771Z [INFO] server-554072fc-a8a1-bf41-8ac6-8596225fc000: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 Address:e35886ff-b3c4-2beb-6f96-2bf8187ac724} {Suffrage:Voter ID:server-ffbe7595-de26-4eab-9c10-64ed72fc0acf Address:ffbe7595-de26-4eab-9c10-64ed72fc0acf} {Suffrage:Voter ID:server-554072fc-a8a1-bf41-8ac6-8596225fc000 Address:554072fc-a8a1-bf41-8ac6-8596225fc000}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.771Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:57.771Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: 2024-01-08T10:37:57.770Z [INFO] server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: entering follower state: follower="Node at e35886ff-b3c4-2beb-6f96-2bf8187ac724 [Follower]" leader= + testing.go:149: server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: 2024-01-08T10:37:57.770Z [INFO] server-e35886ff-b3c4-2beb-6f96-2bf8187ac724: entering follower state: follower="Node at e35886ff-b3c4-2beb-6f96-2bf8187ac724 [Follower]" leader= +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.771Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering follower state: follower="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Follower]" leader= + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.771Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering follower state: follower="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Follower]" leader= +testLoggerAdapter verbose: server-554072fc-a8a1-bf41-8ac6-8596225fc000: 2024-01-08T10:37:57.772Z [INFO] server-554072fc-a8a1-bf41-8ac6-8596225fc000: entering follower state: follower="Node at 554072fc-a8a1-bf41-8ac6-8596225fc000 [Follower]" leader= + testing.go:149: server-554072fc-a8a1-bf41-8ac6-8596225fc000: 2024-01-08T10:37:57.772Z [INFO] server-554072fc-a8a1-bf41-8ac6-8596225fc000: entering follower state: follower="Node at 554072fc-a8a1-bf41-8ac6-8596225fc000 [Follower]" leader= +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.826Z [WARN] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.826Z [WARN] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.827Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering candidate state: node="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Candidate]" term=2 + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.827Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering candidate state: node="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Candidate]" term=2 +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.827Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: election won: tally=2 + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.827Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: election won: tally=2 +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering leader state: leader="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Leader]" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: entering leader state: leader="Node at ffbe7595-de26-4eab-9c10-64ed72fc0acf [Leader]" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: added peer, starting replication: peer=server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: added peer, starting replication: peer=server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: added peer, starting replication: peer=server-554072fc-a8a1-bf41-8ac6-8596225fc000 + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.828Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: added peer, starting replication: peer=server-554072fc-a8a1-bf41-8ac6-8596225fc000 +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.829Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-554072fc-a8a1-bf41-8ac6-8596225fc000 554072fc-a8a1-bf41-8ac6-8596225fc000}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.829Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-554072fc-a8a1-bf41-8ac6-8596225fc000 554072fc-a8a1-bf41-8ac6-8596225fc000}" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.830Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:57.830Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.935Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.830316701 +0000 UTC m=+5.982110871 (1 nodes), 58.530192ms from start of poll, 72.617841ms from cluster start. Timeout at 2024-01-08 10:37:57.93583383 +0000 UTC m=+6.087627954, 105.517083ms after stability + testing.go:149: cluster: 2024-01-08T10:37:57.935Z [INFO] stable state for Leader reached at 2024-01-08 10:37:57.830316701 +0000 UTC m=+5.982110871 (1 nodes), 58.530192ms from start of poll, 72.617841ms from cluster start. Timeout at 2024-01-08 10:37:57.93583383 +0000 UTC m=+6.087627954, 105.517083ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:57.936Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:37:57.936Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.042Z [INFO] stable state for Follower reached at 2024-01-08 10:37:57.936311447 +0000 UTC m=+6.088105575 (2 nodes), 4.612µs from start of poll, 178.612545ms from cluster start. Timeout at 2024-01-08 10:37:58.0415297 +0000 UTC m=+6.193323817, 105.218242ms after stability + testing.go:149: cluster: 2024-01-08T10:37:58.042Z [INFO] stable state for Follower reached at 2024-01-08 10:37:57.936311447 +0000 UTC m=+6.088105575 (2 nodes), 4.612µs from start of poll, 178.612545ms from cluster start. Timeout at 2024-01-08 10:37:58.0415297 +0000 UTC m=+6.193323817, 105.218242ms after stability +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.042Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to pipeline appendEntries: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" error="append pipeline closed" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.042Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to pipeline appendEntries: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" error="append pipeline closed" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.043Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.043Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.043Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to appendEntries to: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" error="failed to connect to peer: e35886ff-b3c4-2beb-6f96-2bf8187ac724" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.043Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to appendEntries to: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" error="failed to connect to peer: e35886ff-b3c4-2beb-6f96-2bf8187ac724" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.046Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to heartbeat to: peer=e35886ff-b3c4-2beb-6f96-2bf8187ac724 error="failed to connect to peer: e35886ff-b3c4-2beb-6f96-2bf8187ac724" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.046Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to heartbeat to: peer=e35886ff-b3c4-2beb-6f96-2bf8187ac724 error="failed to connect to peer: e35886ff-b3c4-2beb-6f96-2bf8187ac724" raft_test.go:420: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.513Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.513Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: pipelining replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.526Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:11.526Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.632Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.526550674 +0000 UTC m=+5.856783360 (1 nodes), 1.204µs from start of poll, 303.884418ms from cluster start. Timeout at 2025-02-09 16:52:11.632174705 +0000 UTC m=+5.962407398, 105.624038ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.632Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.526550674 +0000 UTC m=+5.856783360 (1 nodes), 1.204µs from start of poll, 303.884418ms from cluster start. Timeout at 2025-02-09 16:52:11.632174705 +0000 UTC m=+5.962407398, 105.624038ms after stability -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.632Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.632Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-77f9f4ee-cb0b-fe10-e2a5-aa304da7d795 77f9f4ee-cb0b-fe10-e2a5-aa304da7d795}" -testLoggerAdapter verbose: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.632Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" - testing.go:149: server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: 2025-02-09T16:52:11.632Z [INFO] server-06ffd059-69aa-e354-e84c-f264a1c5f4d8: aborting pipeline replication: peer="{Voter server-0936aadc-eb10-aaff-a82c-3a2bce0bf5d6 0936aadc-eb10-aaff-a82c-3a2bce0bf5d6}" ---- PASS: TestRaft_BehindFollower (0.41s) +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.062Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.062Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: pipelining replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.081Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:58.081Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.190Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.082272431 +0000 UTC m=+6.234066555 (1 nodes), 3.155µs from start of poll, 324.573525ms from cluster start. Timeout at 2024-01-08 10:37:58.189872873 +0000 UTC m=+6.341666987, 107.600432ms after stability + testing.go:149: cluster: 2024-01-08T10:37:58.190Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.082272431 +0000 UTC m=+6.234066555 (1 nodes), 3.155µs from start of poll, 324.573525ms from cluster start. Timeout at 2024-01-08 10:37:58.189872873 +0000 UTC m=+6.341666987, 107.600432ms after stability +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.190Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.190Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-e35886ff-b3c4-2beb-6f96-2bf8187ac724 e35886ff-b3c4-2beb-6f96-2bf8187ac724}" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.190Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-554072fc-a8a1-bf41-8ac6-8596225fc000 554072fc-a8a1-bf41-8ac6-8596225fc000}" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.190Z [INFO] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: aborting pipeline replication: peer="{Voter server-554072fc-a8a1-bf41-8ac6-8596225fc000 554072fc-a8a1-bf41-8ac6-8596225fc000}" +testLoggerAdapter verbose: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.241Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to heartbeat to: peer=e35886ff-b3c4-2beb-6f96-2bf8187ac724 error="command timed out" + testing.go:149: server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: 2024-01-08T10:37:58.241Z [ERROR] server-ffbe7595-de26-4eab-9c10-64ed72fc0acf: failed to heartbeat to: peer=e35886ff-b3c4-2beb-6f96-2bf8187ac724 error="command timed out" +--- PASS: TestRaft_BehindFollower (0.49s) === RUN TestRaft_ApplyNonLeader -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.633Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.633Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" -testLoggerAdapter verbose: server-65d53bb2-3b27-bd36-2a22-df5356e7903c: 2025-02-09T16:52:11.633Z [INFO] server-65d53bb2-3b27-bd36-2a22-df5356e7903c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" - testing.go:149: server-65d53bb2-3b27-bd36-2a22-df5356e7903c: 2025-02-09T16:52:11.633Z [INFO] server-65d53bb2-3b27-bd36-2a22-df5356e7903c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.633Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering follower state: follower="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Follower]" leader= -testLoggerAdapter verbose: server-c15fe453-254e-c06d-67f8-752fc90a3b1b: 2025-02-09T16:52:11.633Z [INFO] server-c15fe453-254e-c06d-67f8-752fc90a3b1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.633Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering follower state: follower="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Follower]" leader= - testing.go:149: server-c15fe453-254e-c06d-67f8-752fc90a3b1b: 2025-02-09T16:52:11.633Z [INFO] server-c15fe453-254e-c06d-67f8-752fc90a3b1b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2d5e8375-71bb-338d-d0d6-0d8c4e571953 Address:2d5e8375-71bb-338d-d0d6-0d8c4e571953} {Suffrage:Voter ID:server-65d53bb2-3b27-bd36-2a22-df5356e7903c Address:65d53bb2-3b27-bd36-2a22-df5356e7903c} {Suffrage:Voter ID:server-c15fe453-254e-c06d-67f8-752fc90a3b1b Address:c15fe453-254e-c06d-67f8-752fc90a3b1b}]" -testLoggerAdapter verbose: server-65d53bb2-3b27-bd36-2a22-df5356e7903c: 2025-02-09T16:52:11.633Z [INFO] server-65d53bb2-3b27-bd36-2a22-df5356e7903c: entering follower state: follower="Node at 65d53bb2-3b27-bd36-2a22-df5356e7903c [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.633Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-65d53bb2-3b27-bd36-2a22-df5356e7903c: 2025-02-09T16:52:11.633Z [INFO] server-65d53bb2-3b27-bd36-2a22-df5356e7903c: entering follower state: follower="Node at 65d53bb2-3b27-bd36-2a22-df5356e7903c [Follower]" leader= - testing.go:149: cluster: 2025-02-09T16:52:11.633Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c15fe453-254e-c06d-67f8-752fc90a3b1b: 2025-02-09T16:52:11.633Z [INFO] server-c15fe453-254e-c06d-67f8-752fc90a3b1b: entering follower state: follower="Node at c15fe453-254e-c06d-67f8-752fc90a3b1b [Follower]" leader= - testing.go:149: server-c15fe453-254e-c06d-67f8-752fc90a3b1b: 2025-02-09T16:52:11.633Z [INFO] server-c15fe453-254e-c06d-67f8-752fc90a3b1b: entering follower state: follower="Node at c15fe453-254e-c06d-67f8-752fc90a3b1b [Follower]" leader= -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [WARN] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [WARN] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering candidate state: node="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Candidate]" term=2 - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering candidate state: node="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Candidate]" term=2 -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: election won: tally=2 - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.696Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: election won: tally=2 -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering leader state: leader="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Leader]" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: entering leader state: leader="Node at 2d5e8375-71bb-338d-d0d6-0d8c4e571953 [Leader]" -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: added peer, starting replication: peer=server-65d53bb2-3b27-bd36-2a22-df5356e7903c - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: added peer, starting replication: peer=server-65d53bb2-3b27-bd36-2a22-df5356e7903c -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: added peer, starting replication: peer=server-c15fe453-254e-c06d-67f8-752fc90a3b1b - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.697Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: added peer, starting replication: peer=server-c15fe453-254e-c06d-67f8-752fc90a3b1b -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.698Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: pipelining replication: peer="{Voter server-c15fe453-254e-c06d-67f8-752fc90a3b1b c15fe453-254e-c06d-67f8-752fc90a3b1b}" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.698Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: pipelining replication: peer="{Voter server-c15fe453-254e-c06d-67f8-752fc90a3b1b c15fe453-254e-c06d-67f8-752fc90a3b1b}" -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.698Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: pipelining replication: peer="{Voter server-65d53bb2-3b27-bd36-2a22-df5356e7903c 65d53bb2-3b27-bd36-2a22-df5356e7903c}" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.698Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: pipelining replication: peer="{Voter server-65d53bb2-3b27-bd36-2a22-df5356e7903c 65d53bb2-3b27-bd36-2a22-df5356e7903c}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.803Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.698141449 +0000 UTC m=+6.028374139 (1 nodes), 64.642216ms from start of poll, 65.104733ms from cluster start. Timeout at 2025-02-09 16:52:11.803796472 +0000 UTC m=+6.134029160, 105.655021ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.803Z [INFO] stable state for Leader reached at 2025-02-09 16:52:11.698141449 +0000 UTC m=+6.028374139 (1 nodes), 64.642216ms from start of poll, 65.104733ms from cluster start. Timeout at 2025-02-09 16:52:11.803796472 +0000 UTC m=+6.134029160, 105.655021ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.803Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:11.803Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.909Z [INFO] stable state for Follower reached at 2025-02-09 16:52:11.803940528 +0000 UTC m=+6.134173215 (2 nodes), 2.04µs from start of poll, 170.903809ms from cluster start. Timeout at 2025-02-09 16:52:11.909885156 +0000 UTC m=+6.240117844, 105.944629ms after stability - testing.go:149: cluster: 2025-02-09T16:52:11.909Z [INFO] stable state for Follower reached at 2025-02-09 16:52:11.803940528 +0000 UTC m=+6.134173215 (2 nodes), 2.04µs from start of poll, 170.903809ms from cluster start. Timeout at 2025-02-09 16:52:11.909885156 +0000 UTC m=+6.240117844, 105.944629ms after stability -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.910Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: aborting pipeline replication: peer="{Voter server-c15fe453-254e-c06d-67f8-752fc90a3b1b c15fe453-254e-c06d-67f8-752fc90a3b1b}" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.910Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: aborting pipeline replication: peer="{Voter server-c15fe453-254e-c06d-67f8-752fc90a3b1b c15fe453-254e-c06d-67f8-752fc90a3b1b}" -testLoggerAdapter verbose: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.910Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: aborting pipeline replication: peer="{Voter server-65d53bb2-3b27-bd36-2a22-df5356e7903c 65d53bb2-3b27-bd36-2a22-df5356e7903c}" - testing.go:149: server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: 2025-02-09T16:52:11.910Z [INFO] server-2d5e8375-71bb-338d-d0d6-0d8c4e571953: aborting pipeline replication: peer="{Voter server-65d53bb2-3b27-bd36-2a22-df5356e7903c 65d53bb2-3b27-bd36-2a22-df5356e7903c}" ---- PASS: TestRaft_ApplyNonLeader (0.28s) +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.250Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.250Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" +testLoggerAdapter verbose: server-9069da43-9162-0980-805f-53736fbd3fb2: 2024-01-08T10:37:58.251Z [INFO] server-9069da43-9162-0980-805f-53736fbd3fb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" + testing.go:149: server-9069da43-9162-0980-805f-53736fbd3fb2: 2024-01-08T10:37:58.251Z [INFO] server-9069da43-9162-0980-805f-53736fbd3fb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" +testLoggerAdapter verbose: server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: 2024-01-08T10:37:58.252Z [INFO] server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" + testing.go:149: server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: 2024-01-08T10:37:58.252Z [INFO] server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d7c43658-63e3-7733-1bec-90adffb6b5e4 Address:d7c43658-63e3-7733-1bec-90adffb6b5e4} {Suffrage:Voter ID:server-9069da43-9162-0980-805f-53736fbd3fb2 Address:9069da43-9162-0980-805f-53736fbd3fb2} {Suffrage:Voter ID:server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f Address:ba3b92fc-f488-8ff5-350c-9c6185a43d1f}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.252Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:58.252Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.252Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering follower state: follower="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Follower]" leader= + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.252Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering follower state: follower="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Follower]" leader= +testLoggerAdapter verbose: server-9069da43-9162-0980-805f-53736fbd3fb2: 2024-01-08T10:37:58.252Z [INFO] server-9069da43-9162-0980-805f-53736fbd3fb2: entering follower state: follower="Node at 9069da43-9162-0980-805f-53736fbd3fb2 [Follower]" leader= + testing.go:149: server-9069da43-9162-0980-805f-53736fbd3fb2: 2024-01-08T10:37:58.252Z [INFO] server-9069da43-9162-0980-805f-53736fbd3fb2: entering follower state: follower="Node at 9069da43-9162-0980-805f-53736fbd3fb2 [Follower]" leader= +testLoggerAdapter verbose: server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: 2024-01-08T10:37:58.253Z [INFO] server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: entering follower state: follower="Node at ba3b92fc-f488-8ff5-350c-9c6185a43d1f [Follower]" leader= + testing.go:149: server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: 2024-01-08T10:37:58.253Z [INFO] server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f: entering follower state: follower="Node at ba3b92fc-f488-8ff5-350c-9c6185a43d1f [Follower]" leader= +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [WARN] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [WARN] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering candidate state: node="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Candidate]" term=2 + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering candidate state: node="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Candidate]" term=2 +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: election won: tally=2 + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.318Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: election won: tally=2 +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering leader state: leader="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Leader]" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: entering leader state: leader="Node at d7c43658-63e3-7733-1bec-90adffb6b5e4 [Leader]" +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: added peer, starting replication: peer=server-9069da43-9162-0980-805f-53736fbd3fb2 + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: added peer, starting replication: peer=server-9069da43-9162-0980-805f-53736fbd3fb2 +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: added peer, starting replication: peer=server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: added peer, starting replication: peer=server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: pipelining replication: peer="{Voter server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f ba3b92fc-f488-8ff5-350c-9c6185a43d1f}" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.319Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: pipelining replication: peer="{Voter server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f ba3b92fc-f488-8ff5-350c-9c6185a43d1f}" +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.320Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: pipelining replication: peer="{Voter server-9069da43-9162-0980-805f-53736fbd3fb2 9069da43-9162-0980-805f-53736fbd3fb2}" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.320Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: pipelining replication: peer="{Voter server-9069da43-9162-0980-805f-53736fbd3fb2 9069da43-9162-0980-805f-53736fbd3fb2}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.426Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.320746129 +0000 UTC m=+6.472540287 (1 nodes), 68.259292ms from start of poll, 75.050338ms from cluster start. Timeout at 2024-01-08 10:37:58.426115722 +0000 UTC m=+6.577909846, 105.369559ms after stability + testing.go:149: cluster: 2024-01-08T10:37:58.426Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.320746129 +0000 UTC m=+6.472540287 (1 nodes), 68.259292ms from start of poll, 75.050338ms from cluster start. Timeout at 2024-01-08 10:37:58.426115722 +0000 UTC m=+6.577909846, 105.369559ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.426Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:37:58.426Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.542Z [INFO] stable state for Follower reached at 2024-01-08 10:37:58.426609313 +0000 UTC m=+6.578403436 (2 nodes), 4.519µs from start of poll, 180.913487ms from cluster start. Timeout at 2024-01-08 10:37:58.541898863 +0000 UTC m=+6.693692982, 115.289546ms after stability + testing.go:149: cluster: 2024-01-08T10:37:58.542Z [INFO] stable state for Follower reached at 2024-01-08 10:37:58.426609313 +0000 UTC m=+6.578403436 (2 nodes), 4.519µs from start of poll, 180.913487ms from cluster start. Timeout at 2024-01-08 10:37:58.541898863 +0000 UTC m=+6.693692982, 115.289546ms after stability +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.542Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: aborting pipeline replication: peer="{Voter server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f ba3b92fc-f488-8ff5-350c-9c6185a43d1f}" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.542Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: aborting pipeline replication: peer="{Voter server-ba3b92fc-f488-8ff5-350c-9c6185a43d1f ba3b92fc-f488-8ff5-350c-9c6185a43d1f}" +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.542Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: aborting pipeline replication: peer="{Voter server-9069da43-9162-0980-805f-53736fbd3fb2 9069da43-9162-0980-805f-53736fbd3fb2}" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.542Z [INFO] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: aborting pipeline replication: peer="{Voter server-9069da43-9162-0980-805f-53736fbd3fb2 9069da43-9162-0980-805f-53736fbd3fb2}" +testLoggerAdapter verbose: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.593Z [ERROR] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: failed to heartbeat to: peer=9069da43-9162-0980-805f-53736fbd3fb2 error="command timed out" + testing.go:149: server-d7c43658-63e3-7733-1bec-90adffb6b5e4: 2024-01-08T10:37:58.593Z [ERROR] server-d7c43658-63e3-7733-1bec-90adffb6b5e4: failed to heartbeat to: peer=9069da43-9162-0980-805f-53736fbd3fb2 error="command timed out" +--- PASS: TestRaft_ApplyNonLeader (0.35s) === RUN TestRaft_ApplyConcurrent -testLoggerAdapter verbose: server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: 2025-02-09T16:52:11.910Z [INFO] server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" - testing.go:149: server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: 2025-02-09T16:52:11.910Z [INFO] server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:11.910Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" -testLoggerAdapter verbose: server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: 2025-02-09T16:52:11.910Z [INFO] server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: entering follower state: follower="Node at 31553d2c-8960-b9c9-0e16-ef8f8be4c95e [Follower]" leader= - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:11.910Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:11.910Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering follower state: follower="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Follower]" leader= - testing.go:149: server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: 2025-02-09T16:52:11.910Z [INFO] server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e: entering follower state: follower="Node at 31553d2c-8960-b9c9-0e16-ef8f8be4c95e [Follower]" leader= - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:11.910Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering follower state: follower="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Follower]" leader= -testLoggerAdapter verbose: server-8c62e9ff-2000-1e06-839f-4e47f980fde0: 2025-02-09T16:52:11.911Z [INFO] server-8c62e9ff-2000-1e06-839f-4e47f980fde0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" - testing.go:149: server-8c62e9ff-2000-1e06-839f-4e47f980fde0: 2025-02-09T16:52:11.911Z [INFO] server-8c62e9ff-2000-1e06-839f-4e47f980fde0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e Address:31553d2c-8960-b9c9-0e16-ef8f8be4c95e} {Suffrage:Voter ID:server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d Address:798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d} {Suffrage:Voter ID:server-8c62e9ff-2000-1e06-839f-4e47f980fde0 Address:8c62e9ff-2000-1e06-839f-4e47f980fde0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:11.911Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:11.911Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8c62e9ff-2000-1e06-839f-4e47f980fde0: 2025-02-09T16:52:11.911Z [INFO] server-8c62e9ff-2000-1e06-839f-4e47f980fde0: entering follower state: follower="Node at 8c62e9ff-2000-1e06-839f-4e47f980fde0 [Follower]" leader= - testing.go:149: server-8c62e9ff-2000-1e06-839f-4e47f980fde0: 2025-02-09T16:52:11.911Z [INFO] server-8c62e9ff-2000-1e06-839f-4e47f980fde0: entering follower state: follower="Node at 8c62e9ff-2000-1e06-839f-4e47f980fde0 [Follower]" leader= -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [WARN] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [WARN] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering candidate state: node="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Candidate]" term=2 - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering candidate state: node="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Candidate]" term=2 -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: election won: tally=2 - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: election won: tally=2 -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering leader state: leader="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Leader]" - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: entering leader state: leader="Node at 798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d [Leader]" -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: added peer, starting replication: peer=server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: added peer, starting replication: peer=server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: added peer, starting replication: peer=server-8c62e9ff-2000-1e06-839f-4e47f980fde0 - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: added peer, starting replication: peer=server-8c62e9ff-2000-1e06-839f-4e47f980fde0 -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: pipelining replication: peer="{Voter server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e 31553d2c-8960-b9c9-0e16-ef8f8be4c95e}" - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: pipelining replication: peer="{Voter server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e 31553d2c-8960-b9c9-0e16-ef8f8be4c95e}" -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: pipelining replication: peer="{Voter server-8c62e9ff-2000-1e06-839f-4e47f980fde0 8c62e9ff-2000-1e06-839f-4e47f980fde0}" - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.024Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: pipelining replication: peer="{Voter server-8c62e9ff-2000-1e06-839f-4e47f980fde0 8c62e9ff-2000-1e06-839f-4e47f980fde0}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.230Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.024787828 +0000 UTC m=+6.355020524 (1 nodes), 113.567041ms from start of poll, 114.086659ms from cluster start. Timeout at 2025-02-09 16:52:12.230095105 +0000 UTC m=+6.560327793, 205.307269ms after stability - testing.go:149: cluster: 2025-02-09T16:52:12.230Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.024787828 +0000 UTC m=+6.355020524 (1 nodes), 113.567041ms from start of poll, 114.086659ms from cluster start. Timeout at 2025-02-09 16:52:12.230095105 +0000 UTC m=+6.560327793, 205.307269ms after stability -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.243Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: aborting pipeline replication: peer="{Voter server-8c62e9ff-2000-1e06-839f-4e47f980fde0 8c62e9ff-2000-1e06-839f-4e47f980fde0}" - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.243Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: aborting pipeline replication: peer="{Voter server-8c62e9ff-2000-1e06-839f-4e47f980fde0 8c62e9ff-2000-1e06-839f-4e47f980fde0}" -testLoggerAdapter verbose: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.243Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: aborting pipeline replication: peer="{Voter server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e 31553d2c-8960-b9c9-0e16-ef8f8be4c95e}" - testing.go:149: server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: 2025-02-09T16:52:12.243Z [INFO] server-798b00cf-1457-f24e-e0ce-f0f8fc3e9d3d: aborting pipeline replication: peer="{Voter server-31553d2c-8960-b9c9-0e16-ef8f8be4c95e 31553d2c-8960-b9c9-0e16-ef8f8be4c95e}" ---- PASS: TestRaft_ApplyConcurrent (0.33s) +testLoggerAdapter verbose: server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: 2024-01-08T10:37:58.594Z [INFO] server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" + testing.go:149: server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: 2024-01-08T10:37:58.594Z [INFO] server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" +testLoggerAdapter verbose: server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: 2024-01-08T10:37:58.595Z [INFO] server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" + testing.go:149: server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: 2024-01-08T10:37:58.595Z [INFO] server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.595Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.595Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 Address:166a5aa7-ac90-ae61-3cd7-834bf1b9af29} {Suffrage:Voter ID:server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf Address:6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf} {Suffrage:Voter ID:server-63826fad-7260-472a-c1f5-832dec34bd74 Address:63826fad-7260-472a-c1f5-832dec34bd74}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.595Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:58.595Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: 2024-01-08T10:37:58.595Z [INFO] server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: entering follower state: follower="Node at 166a5aa7-ac90-ae61-3cd7-834bf1b9af29 [Follower]" leader= + testing.go:149: server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: 2024-01-08T10:37:58.595Z [INFO] server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29: entering follower state: follower="Node at 166a5aa7-ac90-ae61-3cd7-834bf1b9af29 [Follower]" leader= +testLoggerAdapter verbose: server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: 2024-01-08T10:37:58.595Z [INFO] server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: entering follower state: follower="Node at 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf [Follower]" leader= + testing.go:149: server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: 2024-01-08T10:37:58.595Z [INFO] server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf: entering follower state: follower="Node at 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf [Follower]" leader= +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.595Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering follower state: follower="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Follower]" leader= + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.595Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering follower state: follower="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Follower]" leader= +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [WARN] server-63826fad-7260-472a-c1f5-832dec34bd74: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [WARN] server-63826fad-7260-472a-c1f5-832dec34bd74: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering candidate state: node="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Candidate]" term=2 + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering candidate state: node="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Candidate]" term=2 +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: election won: tally=2 + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: election won: tally=2 +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering leader state: leader="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Leader]" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: entering leader state: leader="Node at 63826fad-7260-472a-c1f5-832dec34bd74 [Leader]" +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: added peer, starting replication: peer=server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: added peer, starting replication: peer=server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: added peer, starting replication: peer=server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: added peer, starting replication: peer=server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: pipelining replication: peer="{Voter server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf}" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.705Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: pipelining replication: peer="{Voter server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf}" +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.706Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: pipelining replication: peer="{Voter server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 166a5aa7-ac90-ae61-3cd7-834bf1b9af29}" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.706Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: pipelining replication: peer="{Voter server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 166a5aa7-ac90-ae61-3cd7-834bf1b9af29}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.914Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.706240295 +0000 UTC m=+6.858034449 (1 nodes), 110.99489ms from start of poll, 111.524995ms from cluster start. Timeout at 2024-01-08 10:37:58.913885085 +0000 UTC m=+7.065679209, 207.64476ms after stability + testing.go:149: cluster: 2024-01-08T10:37:58.914Z [INFO] stable state for Leader reached at 2024-01-08 10:37:58.706240295 +0000 UTC m=+6.858034449 (1 nodes), 110.99489ms from start of poll, 111.524995ms from cluster start. Timeout at 2024-01-08 10:37:58.913885085 +0000 UTC m=+7.065679209, 207.64476ms after stability +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.986Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: aborting pipeline replication: peer="{Voter server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 166a5aa7-ac90-ae61-3cd7-834bf1b9af29}" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.986Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: aborting pipeline replication: peer="{Voter server-166a5aa7-ac90-ae61-3cd7-834bf1b9af29 166a5aa7-ac90-ae61-3cd7-834bf1b9af29}" +testLoggerAdapter verbose: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.986Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: aborting pipeline replication: peer="{Voter server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf}" + testing.go:149: server-63826fad-7260-472a-c1f5-832dec34bd74: 2024-01-08T10:37:58.986Z [INFO] server-63826fad-7260-472a-c1f5-832dec34bd74: aborting pipeline replication: peer="{Voter server-6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf 6254d8e8-e4ca-df41-ac3c-5f4e4e4b00cf}" +--- PASS: TestRaft_ApplyConcurrent (0.39s) === RUN TestRaft_ApplyConcurrent_Timeout -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.243Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45951805-526c-c287-56e1-44c246e01f0a Address:45951805-526c-c287-56e1-44c246e01f0a}]" - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.243Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-45951805-526c-c287-56e1-44c246e01f0a Address:45951805-526c-c287-56e1-44c246e01f0a}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.243Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:12.243Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.244Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering follower state: follower="Node at 45951805-526c-c287-56e1-44c246e01f0a [Follower]" leader= - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.244Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering follower state: follower="Node at 45951805-526c-c287-56e1-44c246e01f0a [Follower]" leader= -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [WARN] server-45951805-526c-c287-56e1-44c246e01f0a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [WARN] server-45951805-526c-c287-56e1-44c246e01f0a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering candidate state: node="Node at 45951805-526c-c287-56e1-44c246e01f0a [Candidate]" term=2 - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering candidate state: node="Node at 45951805-526c-c287-56e1-44c246e01f0a [Candidate]" term=2 -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: election won: tally=1 - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: election won: tally=1 -testLoggerAdapter verbose: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering leader state: leader="Node at 45951805-526c-c287-56e1-44c246e01f0a [Leader]" - testing.go:149: server-45951805-526c-c287-56e1-44c246e01f0a: 2025-02-09T16:52:12.357Z [INFO] server-45951805-526c-c287-56e1-44c246e01f0a: entering leader state: leader="Node at 45951805-526c-c287-56e1-44c246e01f0a [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.558Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.357633314 +0000 UTC m=+6.687866004 (1 nodes), 113.624345ms from start of poll, 113.827876ms from cluster start. Timeout at 2025-02-09 16:52:12.558857083 +0000 UTC m=+6.889089774, 201.22377ms after stability - testing.go:149: cluster: 2025-02-09T16:52:12.558Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.357633314 +0000 UTC m=+6.687866004 (1 nodes), 113.624345ms from start of poll, 113.827876ms from cluster start. Timeout at 2025-02-09 16:52:12.558857083 +0000 UTC m=+6.889089774, 201.22377ms after stability ---- PASS: TestRaft_ApplyConcurrent_Timeout (0.32s) +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:58.990Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63ce1f14-adac-1038-0292-03e7961d66e8 Address:63ce1f14-adac-1038-0292-03e7961d66e8}]" + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:58.990Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-63ce1f14-adac-1038-0292-03e7961d66e8 Address:63ce1f14-adac-1038-0292-03e7961d66e8}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:58.990Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:58.990Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:58.990Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering follower state: follower="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Follower]" leader= + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:58.990Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering follower state: follower="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Follower]" leader= +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [WARN] server-63ce1f14-adac-1038-0292-03e7961d66e8: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [WARN] server-63ce1f14-adac-1038-0292-03e7961d66e8: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering candidate state: node="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Candidate]" term=2 + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering candidate state: node="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Candidate]" term=2 +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: election won: tally=1 + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: election won: tally=1 +testLoggerAdapter verbose: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering leader state: leader="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Leader]" + testing.go:149: server-63ce1f14-adac-1038-0292-03e7961d66e8: 2024-01-08T10:37:59.122Z [INFO] server-63ce1f14-adac-1038-0292-03e7961d66e8: entering leader state: leader="Node at 63ce1f14-adac-1038-0292-03e7961d66e8 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.326Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.123103814 +0000 UTC m=+7.274897952 (1 nodes), 132.677798ms from start of poll, 133.88072ms from cluster start. Timeout at 2024-01-08 10:37:59.325924073 +0000 UTC m=+7.477718211, 202.820259ms after stability + testing.go:149: cluster: 2024-01-08T10:37:59.326Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.123103814 +0000 UTC m=+7.274897952 (1 nodes), 132.677798ms from start of poll, 133.88072ms from cluster start. Timeout at 2024-01-08 10:37:59.325924073 +0000 UTC m=+7.477718211, 202.820259ms after stability +--- PASS: TestRaft_ApplyConcurrent_Timeout (0.34s) === RUN TestRaft_JoinNode -testLoggerAdapter verbose: server-ec146fd5-f24a-ea53-7987-157c5579fbd6: 2025-02-09T16:52:12.560Z [INFO] server-ec146fd5-f24a-ea53-7987-157c5579fbd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7}]" - testing.go:149: server-ec146fd5-f24a-ea53-7987-157c5579fbd6: 2025-02-09T16:52:12.560Z [INFO] server-ec146fd5-f24a-ea53-7987-157c5579fbd6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7}]" -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.561Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7}]" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.561Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7}]" -testLoggerAdapter verbose: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.561Z [INFO] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: initial configuration: index=0 servers=[] - testing.go:149: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.561Z [INFO] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.561Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:12.561Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ec146fd5-f24a-ea53-7987-157c5579fbd6: 2025-02-09T16:52:12.561Z [INFO] server-ec146fd5-f24a-ea53-7987-157c5579fbd6: entering follower state: follower="Node at ec146fd5-f24a-ea53-7987-157c5579fbd6 [Follower]" leader= - testing.go:149: server-ec146fd5-f24a-ea53-7987-157c5579fbd6: 2025-02-09T16:52:12.561Z [INFO] server-ec146fd5-f24a-ea53-7987-157c5579fbd6: entering follower state: follower="Node at ec146fd5-f24a-ea53-7987-157c5579fbd6 [Follower]" leader= -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.561Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering follower state: follower="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Follower]" leader= - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.561Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering follower state: follower="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Follower]" leader= -testLoggerAdapter verbose: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.561Z [INFO] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: entering follower state: follower="Node at ce866238-52b5-8e62-c1cf-dc3a7b284911 [Follower]" leader= - testing.go:149: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.561Z [INFO] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: entering follower state: follower="Node at ce866238-52b5-8e62-c1cf-dc3a7b284911 [Follower]" leader= -testLoggerAdapter verbose: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.615Z [WARN] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: no known peers, aborting election - testing.go:149: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.615Z [WARN] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: no known peers, aborting election -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.626Z [WARN] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.626Z [WARN] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.627Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering candidate state: node="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Candidate]" term=2 - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.627Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering candidate state: node="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Candidate]" term=2 -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: election won: tally=2 - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: election won: tally=2 -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering leader state: leader="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Leader]" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: entering leader state: leader="Node at 7a479977-a20d-2fb7-51b1-b45fe704bfd7 [Leader]" -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: added peer, starting replication: peer=server-ec146fd5-f24a-ea53-7987-157c5579fbd6 - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: added peer, starting replication: peer=server-ec146fd5-f24a-ea53-7987-157c5579fbd6 -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: pipelining replication: peer="{Voter server-ec146fd5-f24a-ea53-7987-157c5579fbd6 ec146fd5-f24a-ea53-7987-157c5579fbd6}" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.628Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: pipelining replication: peer="{Voter server-ec146fd5-f24a-ea53-7987-157c5579fbd6 ec146fd5-f24a-ea53-7987-157c5579fbd6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.734Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.628938631 +0000 UTC m=+6.959171326 (1 nodes), 67.613522ms from start of poll, 68.112593ms from cluster start. Timeout at 2025-02-09 16:52:12.734876769 +0000 UTC m=+7.065109462, 105.938136ms after stability - testing.go:149: cluster: 2025-02-09T16:52:12.734Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.628938631 +0000 UTC m=+6.959171326 (1 nodes), 67.613522ms from start of poll, 68.112593ms from cluster start. Timeout at 2025-02-09 16:52:12.734876769 +0000 UTC m=+7.065109462, 105.938136ms after stability -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: updating configuration: command=AddStaging server-id=server-ce866238-52b5-8e62-c1cf-dc3a7b284911 server-addr=ce866238-52b5-8e62-c1cf-dc3a7b284911 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7} {Suffrage:Voter ID:server-ce866238-52b5-8e62-c1cf-dc3a7b284911 Address:ce866238-52b5-8e62-c1cf-dc3a7b284911}]" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: updating configuration: command=AddStaging server-id=server-ce866238-52b5-8e62-c1cf-dc3a7b284911 server-addr=ce866238-52b5-8e62-c1cf-dc3a7b284911 servers="[{Suffrage:Voter ID:server-ec146fd5-f24a-ea53-7987-157c5579fbd6 Address:ec146fd5-f24a-ea53-7987-157c5579fbd6} {Suffrage:Voter ID:server-7a479977-a20d-2fb7-51b1-b45fe704bfd7 Address:7a479977-a20d-2fb7-51b1-b45fe704bfd7} {Suffrage:Voter ID:server-ce866238-52b5-8e62-c1cf-dc3a7b284911 Address:ce866238-52b5-8e62-c1cf-dc3a7b284911}]" -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: added peer, starting replication: peer=server-ce866238-52b5-8e62-c1cf-dc3a7b284911 - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: added peer, starting replication: peer=server-ce866238-52b5-8e62-c1cf-dc3a7b284911 -testLoggerAdapter verbose: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.735Z [WARN] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-ce866238-52b5-8e62-c1cf-dc3a7b284911: 2025-02-09T16:52:12.735Z [WARN] server-ce866238-52b5-8e62-c1cf-dc3a7b284911: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [WARN] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: appendEntries rejected, sending older logs: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" next=1 - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [WARN] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: appendEntries rejected, sending older logs: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" next=1 -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: pipelining replication: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.735Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: pipelining replication: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.735Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:12.735Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.841Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.735517657 +0000 UTC m=+7.065750346 (1 nodes), 651ns from start of poll, 174.691613ms from cluster start. Timeout at 2025-02-09 16:52:12.841236795 +0000 UTC m=+7.171469486, 105.71914ms after stability - testing.go:149: cluster: 2025-02-09T16:52:12.841Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.735517657 +0000 UTC m=+7.065750346 (1 nodes), 651ns from start of poll, 174.691613ms from cluster start. Timeout at 2025-02-09 16:52:12.841236795 +0000 UTC m=+7.171469486, 105.71914ms after stability -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.841Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: aborting pipeline replication: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.841Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: aborting pipeline replication: peer="{Voter server-ce866238-52b5-8e62-c1cf-dc3a7b284911 ce866238-52b5-8e62-c1cf-dc3a7b284911}" -testLoggerAdapter verbose: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.841Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: aborting pipeline replication: peer="{Voter server-ec146fd5-f24a-ea53-7987-157c5579fbd6 ec146fd5-f24a-ea53-7987-157c5579fbd6}" - testing.go:149: server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: 2025-02-09T16:52:12.841Z [INFO] server-7a479977-a20d-2fb7-51b1-b45fe704bfd7: aborting pipeline replication: peer="{Voter server-ec146fd5-f24a-ea53-7987-157c5579fbd6 ec146fd5-f24a-ea53-7987-157c5579fbd6}" ---- PASS: TestRaft_JoinNode (0.28s) +testLoggerAdapter verbose: server-feb85222-5296-bfa0-0383-89ba856dda79: 2024-01-08T10:37:59.330Z [INFO] server-feb85222-5296-bfa0-0383-89ba856dda79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70}]" + testing.go:149: server-feb85222-5296-bfa0-0383-89ba856dda79: 2024-01-08T10:37:59.330Z [INFO] server-feb85222-5296-bfa0-0383-89ba856dda79: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70}]" +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.330Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70}]" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.330Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70}]" +testLoggerAdapter verbose: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.331Z [INFO] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: initial configuration: index=0 servers=[] + testing.go:149: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.331Z [INFO] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.331Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:59.331Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-feb85222-5296-bfa0-0383-89ba856dda79: 2024-01-08T10:37:59.331Z [INFO] server-feb85222-5296-bfa0-0383-89ba856dda79: entering follower state: follower="Node at feb85222-5296-bfa0-0383-89ba856dda79 [Follower]" leader= + testing.go:149: server-feb85222-5296-bfa0-0383-89ba856dda79: 2024-01-08T10:37:59.331Z [INFO] server-feb85222-5296-bfa0-0383-89ba856dda79: entering follower state: follower="Node at feb85222-5296-bfa0-0383-89ba856dda79 [Follower]" leader= +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.331Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering follower state: follower="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Follower]" leader= + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.331Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering follower state: follower="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Follower]" leader= +testLoggerAdapter verbose: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.331Z [INFO] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: entering follower state: follower="Node at a4b0e509-d79a-99fe-abbc-798b2ec1a97a [Follower]" leader= + testing.go:149: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.331Z [INFO] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: entering follower state: follower="Node at a4b0e509-d79a-99fe-abbc-798b2ec1a97a [Follower]" leader= +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.404Z [WARN] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.404Z [WARN] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.405Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering candidate state: node="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Candidate]" term=2 + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.405Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering candidate state: node="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Candidate]" term=2 +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.405Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: election won: tally=2 + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.405Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: election won: tally=2 +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering leader state: leader="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Leader]" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: entering leader state: leader="Node at 3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 [Leader]" +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: added peer, starting replication: peer=server-feb85222-5296-bfa0-0383-89ba856dda79 + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: added peer, starting replication: peer=server-feb85222-5296-bfa0-0383-89ba856dda79 +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: pipelining replication: peer="{Voter server-feb85222-5296-bfa0-0383-89ba856dda79 feb85222-5296-bfa0-0383-89ba856dda79}" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.406Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: pipelining replication: peer="{Voter server-feb85222-5296-bfa0-0383-89ba856dda79 feb85222-5296-bfa0-0383-89ba856dda79}" +testLoggerAdapter verbose: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.421Z [WARN] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: no known peers, aborting election + testing.go:149: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.421Z [WARN] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.522Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.40707159 +0000 UTC m=+7.558865732 (1 nodes), 75.847827ms from start of poll, 76.508703ms from cluster start. Timeout at 2024-01-08 10:37:59.521934511 +0000 UTC m=+7.673728630, 114.862898ms after stability + testing.go:149: cluster: 2024-01-08T10:37:59.522Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.40707159 +0000 UTC m=+7.558865732 (1 nodes), 75.847827ms from start of poll, 76.508703ms from cluster start. Timeout at 2024-01-08 10:37:59.521934511 +0000 UTC m=+7.673728630, 114.862898ms after stability +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.522Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: updating configuration: command=AddStaging server-id=server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a server-addr=a4b0e509-d79a-99fe-abbc-798b2ec1a97a servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70} {Suffrage:Voter ID:server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a Address:a4b0e509-d79a-99fe-abbc-798b2ec1a97a}]" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.522Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: updating configuration: command=AddStaging server-id=server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a server-addr=a4b0e509-d79a-99fe-abbc-798b2ec1a97a servers="[{Suffrage:Voter ID:server-feb85222-5296-bfa0-0383-89ba856dda79 Address:feb85222-5296-bfa0-0383-89ba856dda79} {Suffrage:Voter ID:server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70 Address:3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70} {Suffrage:Voter ID:server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a Address:a4b0e509-d79a-99fe-abbc-798b2ec1a97a}]" +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.523Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: added peer, starting replication: peer=server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.523Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: added peer, starting replication: peer=server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a +testLoggerAdapter verbose: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.523Z [WARN] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: 2024-01-08T10:37:59.523Z [WARN] server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.524Z [WARN] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: appendEntries rejected, sending older logs: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" next=1 + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.524Z [WARN] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: appendEntries rejected, sending older logs: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" next=1 +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.524Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: pipelining replication: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.524Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: pipelining replication: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.524Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:59.524Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.634Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.525201917 +0000 UTC m=+7.676996043 (1 nodes), 2.45µs from start of poll, 194.639014ms from cluster start. Timeout at 2024-01-08 10:37:59.633850906 +0000 UTC m=+7.785645044, 108.649001ms after stability + testing.go:149: cluster: 2024-01-08T10:37:59.634Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.525201917 +0000 UTC m=+7.676996043 (1 nodes), 2.45µs from start of poll, 194.639014ms from cluster start. Timeout at 2024-01-08 10:37:59.633850906 +0000 UTC m=+7.785645044, 108.649001ms after stability +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.634Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: aborting pipeline replication: peer="{Voter server-feb85222-5296-bfa0-0383-89ba856dda79 feb85222-5296-bfa0-0383-89ba856dda79}" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.634Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: aborting pipeline replication: peer="{Voter server-feb85222-5296-bfa0-0383-89ba856dda79 feb85222-5296-bfa0-0383-89ba856dda79}" +testLoggerAdapter verbose: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.634Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: aborting pipeline replication: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" + testing.go:149: server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: 2024-01-08T10:37:59.634Z [INFO] server-3cc7c253-7f41-4b47-7f5e-9ae3f81d0f70: aborting pipeline replication: peer="{Voter server-a4b0e509-d79a-99fe-abbc-798b2ec1a97a a4b0e509-d79a-99fe-abbc-798b2ec1a97a}" +--- PASS: TestRaft_JoinNode (0.31s) === RUN TestRaft_JoinNode_ConfigStore -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.842Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9}]" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.842Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9}]" -testLoggerAdapter verbose: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.842Z [INFO] server-6b3ebfb3-99db-5bb7-8125-271801257201: initial configuration: index=0 servers=[] - testing.go:149: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.842Z [INFO] server-6b3ebfb3-99db-5bb7-8125-271801257201: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.842Z [INFO] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: initial configuration: index=0 servers=[] - testing.go:149: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.842Z [INFO] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:12.842Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:12.842Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.842Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering follower state: follower="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Follower]" leader= - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.842Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering follower state: follower="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Follower]" leader= -testLoggerAdapter verbose: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.842Z [INFO] server-6b3ebfb3-99db-5bb7-8125-271801257201: entering follower state: follower="Node at 6b3ebfb3-99db-5bb7-8125-271801257201 [Follower]" leader= - testing.go:149: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.842Z [INFO] server-6b3ebfb3-99db-5bb7-8125-271801257201: entering follower state: follower="Node at 6b3ebfb3-99db-5bb7-8125-271801257201 [Follower]" leader= -testLoggerAdapter verbose: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.842Z [INFO] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: entering follower state: follower="Node at 4786ac2c-2633-9a2e-292b-5891a4d4df85 [Follower]" leader= - testing.go:149: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.842Z [INFO] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: entering follower state: follower="Node at 4786ac2c-2633-9a2e-292b-5891a4d4df85 [Follower]" leader= -testLoggerAdapter verbose: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.922Z [WARN] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: no known peers, aborting election - testing.go:149: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:12.922Z [WARN] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: no known peers, aborting election -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering candidate state: node="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Candidate]" term=2 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering candidate state: node="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Candidate]" term=2 -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: election won: tally=1 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: election won: tally=1 -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering leader state: leader="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Leader]" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:12.931Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: entering leader state: leader="Node at 02d96105-833a-caf2-58f7-620568a4aef9 [Leader]" -testLoggerAdapter verbose: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.943Z [WARN] server-6b3ebfb3-99db-5bb7-8125-271801257201: no known peers, aborting election - testing.go:149: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:12.943Z [WARN] server-6b3ebfb3-99db-5bb7-8125-271801257201: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.037Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.931863158 +0000 UTC m=+7.262095851 (1 nodes), 89.334239ms from start of poll, 89.842925ms from cluster start. Timeout at 2025-02-09 16:52:13.037856352 +0000 UTC m=+7.368089048, 105.993197ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.037Z [INFO] stable state for Leader reached at 2025-02-09 16:52:12.931863158 +0000 UTC m=+7.262095851 (1 nodes), 89.334239ms from start of poll, 89.842925ms from cluster start. Timeout at 2025-02-09 16:52:13.037856352 +0000 UTC m=+7.368089048, 105.993197ms after stability -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.037Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: updating configuration: command=AddStaging server-id=server-6b3ebfb3-99db-5bb7-8125-271801257201 server-addr=6b3ebfb3-99db-5bb7-8125-271801257201 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9} {Suffrage:Voter ID:server-6b3ebfb3-99db-5bb7-8125-271801257201 Address:6b3ebfb3-99db-5bb7-8125-271801257201}]" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.037Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: updating configuration: command=AddStaging server-id=server-6b3ebfb3-99db-5bb7-8125-271801257201 server-addr=6b3ebfb3-99db-5bb7-8125-271801257201 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9} {Suffrage:Voter ID:server-6b3ebfb3-99db-5bb7-8125-271801257201 Address:6b3ebfb3-99db-5bb7-8125-271801257201}]" -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: added peer, starting replication: peer=server-6b3ebfb3-99db-5bb7-8125-271801257201 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: added peer, starting replication: peer=server-6b3ebfb3-99db-5bb7-8125-271801257201 -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.038Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:13.038Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:13.038Z [WARN] server-6b3ebfb3-99db-5bb7-8125-271801257201: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-6b3ebfb3-99db-5bb7-8125-271801257201: 2025-02-09T16:52:13.038Z [WARN] server-6b3ebfb3-99db-5bb7-8125-271801257201: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: appendEntries rejected, sending older logs: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" next=1 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: appendEntries rejected, sending older logs: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" next=1 -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: pipelining replication: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.038Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: pipelining replication: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.143Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.03817071 +0000 UTC m=+7.368403396 (1 nodes), 677ns from start of poll, 196.15047ms from cluster start. Timeout at 2025-02-09 16:52:13.143905243 +0000 UTC m=+7.474137936, 105.73454ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.143Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.03817071 +0000 UTC m=+7.368403396 (1 nodes), 677ns from start of poll, 196.15047ms from cluster start. Timeout at 2025-02-09 16:52:13.143905243 +0000 UTC m=+7.474137936, 105.73454ms after stability -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: updating configuration: command=AddStaging server-id=server-4786ac2c-2633-9a2e-292b-5891a4d4df85 server-addr=4786ac2c-2633-9a2e-292b-5891a4d4df85 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9} {Suffrage:Voter ID:server-6b3ebfb3-99db-5bb7-8125-271801257201 Address:6b3ebfb3-99db-5bb7-8125-271801257201} {Suffrage:Voter ID:server-4786ac2c-2633-9a2e-292b-5891a4d4df85 Address:4786ac2c-2633-9a2e-292b-5891a4d4df85}]" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: updating configuration: command=AddStaging server-id=server-4786ac2c-2633-9a2e-292b-5891a4d4df85 server-addr=4786ac2c-2633-9a2e-292b-5891a4d4df85 servers="[{Suffrage:Voter ID:server-02d96105-833a-caf2-58f7-620568a4aef9 Address:02d96105-833a-caf2-58f7-620568a4aef9} {Suffrage:Voter ID:server-6b3ebfb3-99db-5bb7-8125-271801257201 Address:6b3ebfb3-99db-5bb7-8125-271801257201} {Suffrage:Voter ID:server-4786ac2c-2633-9a2e-292b-5891a4d4df85 Address:4786ac2c-2633-9a2e-292b-5891a4d4df85}]" -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: added peer, starting replication: peer=server-4786ac2c-2633-9a2e-292b-5891a4d4df85 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: added peer, starting replication: peer=server-4786ac2c-2633-9a2e-292b-5891a4d4df85 -testLoggerAdapter verbose: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:13.144Z [WARN] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: failed to get previous log: previous-index=4 last-index=0 error="log not found" - testing.go:149: server-4786ac2c-2633-9a2e-292b-5891a4d4df85: 2025-02-09T16:52:13.144Z [WARN] server-4786ac2c-2633-9a2e-292b-5891a4d4df85: failed to get previous log: previous-index=4 last-index=0 error="log not found" -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: appendEntries rejected, sending older logs: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" next=1 - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [WARN] server-02d96105-833a-caf2-58f7-620568a4aef9: appendEntries rejected, sending older logs: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" next=1 -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: pipelining replication: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.144Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: pipelining replication: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.144Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:13.144Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.250Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.144500799 +0000 UTC m=+7.474733486 (1 nodes), 546ns from start of poll, 302.48056ms from cluster start. Timeout at 2025-02-09 16:52:13.250350559 +0000 UTC m=+7.580583252, 105.849766ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.250Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.144500799 +0000 UTC m=+7.474733486 (1 nodes), 546ns from start of poll, 302.48056ms from cluster start. Timeout at 2025-02-09 16:52:13.250350559 +0000 UTC m=+7.580583252, 105.849766ms after stability -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.250Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: aborting pipeline replication: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.250Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: aborting pipeline replication: peer="{Voter server-4786ac2c-2633-9a2e-292b-5891a4d4df85 4786ac2c-2633-9a2e-292b-5891a4d4df85}" -testLoggerAdapter verbose: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.250Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: aborting pipeline replication: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" - testing.go:149: server-02d96105-833a-caf2-58f7-620568a4aef9: 2025-02-09T16:52:13.250Z [INFO] server-02d96105-833a-caf2-58f7-620568a4aef9: aborting pipeline replication: peer="{Voter server-6b3ebfb3-99db-5bb7-8125-271801257201 6b3ebfb3-99db-5bb7-8125-271801257201}" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.638Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd}]" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.638Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd}]" +testLoggerAdapter verbose: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.639Z [INFO] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: initial configuration: index=0 servers=[] + testing.go:149: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.639Z [INFO] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.640Z [INFO] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: initial configuration: index=0 servers=[] + testing.go:149: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.640Z [INFO] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.640Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:59.640Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.640Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering follower state: follower="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Follower]" leader= + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.640Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering follower state: follower="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Follower]" leader= +testLoggerAdapter verbose: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.640Z [INFO] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: entering follower state: follower="Node at 65330ca5-3335-8d48-175f-81f7a5d07b8e [Follower]" leader= + testing.go:149: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.640Z [INFO] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: entering follower state: follower="Node at 65330ca5-3335-8d48-175f-81f7a5d07b8e [Follower]" leader= +testLoggerAdapter verbose: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.641Z [INFO] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: entering follower state: follower="Node at 3b55b2ef-97da-0f62-e94f-bc736faf72be [Follower]" leader= + testing.go:149: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.641Z [INFO] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: entering follower state: follower="Node at 3b55b2ef-97da-0f62-e94f-bc736faf72be [Follower]" leader= +testLoggerAdapter verbose: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.714Z [WARN] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: no known peers, aborting election + testing.go:149: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.714Z [WARN] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: no known peers, aborting election +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.721Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.721Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering candidate state: node="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Candidate]" term=2 + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering candidate state: node="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Candidate]" term=2 +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: election won: tally=1 + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: election won: tally=1 +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering leader state: leader="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Leader]" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.722Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: entering leader state: leader="Node at f8c2b609-a396-d65e-45f9-3a954f8530bd [Leader]" +testLoggerAdapter verbose: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.722Z [WARN] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: no known peers, aborting election + testing.go:149: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.722Z [WARN] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.830Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.723143436 +0000 UTC m=+7.874937574 (1 nodes), 82.665467ms from start of poll, 85.565863ms from cluster start. Timeout at 2024-01-08 10:37:59.82988355 +0000 UTC m=+7.981677689, 106.740115ms after stability + testing.go:149: cluster: 2024-01-08T10:37:59.830Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.723143436 +0000 UTC m=+7.874937574 (1 nodes), 82.665467ms from start of poll, 85.565863ms from cluster start. Timeout at 2024-01-08 10:37:59.82988355 +0000 UTC m=+7.981677689, 106.740115ms after stability +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.830Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: updating configuration: command=AddStaging server-id=server-65330ca5-3335-8d48-175f-81f7a5d07b8e server-addr=65330ca5-3335-8d48-175f-81f7a5d07b8e servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd} {Suffrage:Voter ID:server-65330ca5-3335-8d48-175f-81f7a5d07b8e Address:65330ca5-3335-8d48-175f-81f7a5d07b8e}]" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.830Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: updating configuration: command=AddStaging server-id=server-65330ca5-3335-8d48-175f-81f7a5d07b8e server-addr=65330ca5-3335-8d48-175f-81f7a5d07b8e servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd} {Suffrage:Voter ID:server-65330ca5-3335-8d48-175f-81f7a5d07b8e Address:65330ca5-3335-8d48-175f-81f7a5d07b8e}]" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.830Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: added peer, starting replication: peer=server-65330ca5-3335-8d48-175f-81f7a5d07b8e + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.830Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: added peer, starting replication: peer=server-65330ca5-3335-8d48-175f-81f7a5d07b8e +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.831Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:59.831Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.831Z [WARN] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-65330ca5-3335-8d48-175f-81f7a5d07b8e: 2024-01-08T10:37:59.831Z [WARN] server-65330ca5-3335-8d48-175f-81f7a5d07b8e: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.831Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: appendEntries rejected, sending older logs: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" next=1 + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.831Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: appendEntries rejected, sending older logs: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" next=1 +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.831Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: pipelining replication: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.831Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: pipelining replication: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.938Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.831216899 +0000 UTC m=+7.983011028 (1 nodes), 2.499µs from start of poll, 193.639317ms from cluster start. Timeout at 2024-01-08 10:37:59.938120845 +0000 UTC m=+8.089914962, 106.903934ms after stability + testing.go:149: cluster: 2024-01-08T10:37:59.938Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.831216899 +0000 UTC m=+7.983011028 (1 nodes), 2.499µs from start of poll, 193.639317ms from cluster start. Timeout at 2024-01-08 10:37:59.938120845 +0000 UTC m=+8.089914962, 106.903934ms after stability +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.938Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: updating configuration: command=AddStaging server-id=server-3b55b2ef-97da-0f62-e94f-bc736faf72be server-addr=3b55b2ef-97da-0f62-e94f-bc736faf72be servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd} {Suffrage:Voter ID:server-65330ca5-3335-8d48-175f-81f7a5d07b8e Address:65330ca5-3335-8d48-175f-81f7a5d07b8e} {Suffrage:Voter ID:server-3b55b2ef-97da-0f62-e94f-bc736faf72be Address:3b55b2ef-97da-0f62-e94f-bc736faf72be}]" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.938Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: updating configuration: command=AddStaging server-id=server-3b55b2ef-97da-0f62-e94f-bc736faf72be server-addr=3b55b2ef-97da-0f62-e94f-bc736faf72be servers="[{Suffrage:Voter ID:server-f8c2b609-a396-d65e-45f9-3a954f8530bd Address:f8c2b609-a396-d65e-45f9-3a954f8530bd} {Suffrage:Voter ID:server-65330ca5-3335-8d48-175f-81f7a5d07b8e Address:65330ca5-3335-8d48-175f-81f7a5d07b8e} {Suffrage:Voter ID:server-3b55b2ef-97da-0f62-e94f-bc736faf72be Address:3b55b2ef-97da-0f62-e94f-bc736faf72be}]" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.938Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: added peer, starting replication: peer=server-3b55b2ef-97da-0f62-e94f-bc736faf72be + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.938Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: added peer, starting replication: peer=server-3b55b2ef-97da-0f62-e94f-bc736faf72be +testLoggerAdapter verbose: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.939Z [WARN] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: failed to get previous log: previous-index=4 last-index=0 error="log not found" + testing.go:149: server-3b55b2ef-97da-0f62-e94f-bc736faf72be: 2024-01-08T10:37:59.939Z [WARN] server-3b55b2ef-97da-0f62-e94f-bc736faf72be: failed to get previous log: previous-index=4 last-index=0 error="log not found" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.939Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: appendEntries rejected, sending older logs: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" next=1 + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.939Z [WARN] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: appendEntries rejected, sending older logs: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" next=1 +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.939Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: pipelining replication: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:37:59.939Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: pipelining replication: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:37:59.940Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:37:59.940Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.046Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.94105412 +0000 UTC m=+8.092848248 (1 nodes), 2.618µs from start of poll, 303.476537ms from cluster start. Timeout at 2024-01-08 10:38:00.046064514 +0000 UTC m=+8.197858656, 105.010408ms after stability + testing.go:149: cluster: 2024-01-08T10:38:00.046Z [INFO] stable state for Leader reached at 2024-01-08 10:37:59.94105412 +0000 UTC m=+8.092848248 (1 nodes), 2.618µs from start of poll, 303.476537ms from cluster start. Timeout at 2024-01-08 10:38:00.046064514 +0000 UTC m=+8.197858656, 105.010408ms after stability +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:38:00.046Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: aborting pipeline replication: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:38:00.046Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: aborting pipeline replication: peer="{Voter server-3b55b2ef-97da-0f62-e94f-bc736faf72be 3b55b2ef-97da-0f62-e94f-bc736faf72be}" +testLoggerAdapter verbose: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:38:00.046Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: aborting pipeline replication: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" + testing.go:149: server-f8c2b609-a396-d65e-45f9-3a954f8530bd: 2024-01-08T10:38:00.046Z [INFO] server-f8c2b609-a396-d65e-45f9-3a954f8530bd: aborting pipeline replication: peer="{Voter server-65330ca5-3335-8d48-175f-81f7a5d07b8e 65330ca5-3335-8d48-175f-81f7a5d07b8e}" --- PASS: TestRaft_JoinNode_ConfigStore (0.41s) === RUN TestRaft_RemoveFollower -testLoggerAdapter verbose: server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: 2025-02-09T16:52:13.251Z [INFO] server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" - testing.go:149: server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: 2025-02-09T16:52:13.251Z [INFO] server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.251Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.251Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" -testLoggerAdapter verbose: server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: 2025-02-09T16:52:13.251Z [INFO] server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" - testing.go:149: server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: 2025-02-09T16:52:13.251Z [INFO] server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 Address:78081e9e-ee36-4aa6-5a88-0e99d26443f5} {Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.251Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:13.251Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: 2025-02-09T16:52:13.251Z [INFO] server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: entering follower state: follower="Node at 78081e9e-ee36-4aa6-5a88-0e99d26443f5 [Follower]" leader= - testing.go:149: server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: 2025-02-09T16:52:13.251Z [INFO] server-78081e9e-ee36-4aa6-5a88-0e99d26443f5: entering follower state: follower="Node at 78081e9e-ee36-4aa6-5a88-0e99d26443f5 [Follower]" leader= -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.251Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering follower state: follower="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Follower]" leader= - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.251Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering follower state: follower="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Follower]" leader= -testLoggerAdapter verbose: server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: 2025-02-09T16:52:13.251Z [INFO] server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: entering follower state: follower="Node at 74c9282a-aa4d-cabb-b17c-d55b9a803c09 [Follower]" leader= - testing.go:149: server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: 2025-02-09T16:52:13.251Z [INFO] server-74c9282a-aa4d-cabb-b17c-d55b9a803c09: entering follower state: follower="Node at 74c9282a-aa4d-cabb-b17c-d55b9a803c09 [Follower]" leader= -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.321Z [WARN] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.321Z [WARN] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.321Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering candidate state: node="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Candidate]" term=2 - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.321Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering candidate state: node="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Candidate]" term=2 -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: election won: tally=2 - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: election won: tally=2 -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering leader state: leader="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Leader]" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: entering leader state: leader="Node at 216e7104-a45e-5dba-0d31-a2c4b44bc119 [Leader]" -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: added peer, starting replication: peer=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: added peer, starting replication: peer=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: added peer, starting replication: peer=server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: added peer, starting replication: peer=server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: pipelining replication: peer="{Voter server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 74c9282a-aa4d-cabb-b17c-d55b9a803c09}" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: pipelining replication: peer="{Voter server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 74c9282a-aa4d-cabb-b17c-d55b9a803c09}" -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: pipelining replication: peer="{Voter server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 78081e9e-ee36-4aa6-5a88-0e99d26443f5}" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.322Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: pipelining replication: peer="{Voter server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 78081e9e-ee36-4aa6-5a88-0e99d26443f5}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.428Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.322467147 +0000 UTC m=+7.652699837 (1 nodes), 70.809037ms from start of poll, 71.249206ms from cluster start. Timeout at 2025-02-09 16:52:13.428412051 +0000 UTC m=+7.758644742, 105.944905ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.428Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.322467147 +0000 UTC m=+7.652699837 (1 nodes), 70.809037ms from start of poll, 71.249206ms from cluster start. Timeout at 2025-02-09 16:52:13.428412051 +0000 UTC m=+7.758644742, 105.944905ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.434Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:13.434Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.539Z [INFO] stable state for Follower reached at 2025-02-09 16:52:13.434166821 +0000 UTC m=+7.764399507 (2 nodes), 1.078µs from start of poll, 182.948876ms from cluster start. Timeout at 2025-02-09 16:52:13.539167788 +0000 UTC m=+7.869400484, 105.000977ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.539Z [INFO] stable state for Follower reached at 2025-02-09 16:52:13.434166821 +0000 UTC m=+7.764399507 (2 nodes), 1.078µs from start of poll, 182.948876ms from cluster start. Timeout at 2025-02-09 16:52:13.539167788 +0000 UTC m=+7.869400484, 105.000977ms after stability -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: updating configuration: command=RemoveServer server-id=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 server-addr= servers="[{Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: updating configuration: command=RemoveServer server-id=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 server-addr= servers="[{Suffrage:Voter ID:server-216e7104-a45e-5dba-0d31-a2c4b44bc119 Address:216e7104-a45e-5dba-0d31-a2c4b44bc119} {Suffrage:Voter ID:server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 Address:74c9282a-aa4d-cabb-b17c-d55b9a803c09}]" -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: removed peer, stopping replication: peer=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 last-index=3 - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: removed peer, stopping replication: peer=server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 last-index=3 -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: aborting pipeline replication: peer="{Voter server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 78081e9e-ee36-4aa6-5a88-0e99d26443f5}" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.539Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: aborting pipeline replication: peer="{Voter server-78081e9e-ee36-4aa6-5a88-0e99d26443f5 78081e9e-ee36-4aa6-5a88-0e99d26443f5}" -testLoggerAdapter verbose: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.644Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: aborting pipeline replication: peer="{Voter server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 74c9282a-aa4d-cabb-b17c-d55b9a803c09}" - testing.go:149: server-216e7104-a45e-5dba-0d31-a2c4b44bc119: 2025-02-09T16:52:13.644Z [INFO] server-216e7104-a45e-5dba-0d31-a2c4b44bc119: aborting pipeline replication: peer="{Voter server-74c9282a-aa4d-cabb-b17c-d55b9a803c09 74c9282a-aa4d-cabb-b17c-d55b9a803c09}" ---- PASS: TestRaft_RemoveFollower (0.39s) +testLoggerAdapter verbose: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.050Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" + testing.go:149: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.050Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" +testLoggerAdapter verbose: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.051Z [INFO] server-22547879-cf85-d863-3037-0bc987d590a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" + testing.go:149: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.051Z [INFO] server-22547879-cf85-d863-3037-0bc987d590a5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.052Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.052Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 Address:84b70fe6-c76d-64db-55d2-6d8ba56b05a0} {Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.052Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:00.052Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.052Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: entering follower state: follower="Node at 84b70fe6-c76d-64db-55d2-6d8ba56b05a0 [Follower]" leader= + testing.go:149: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.052Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: entering follower state: follower="Node at 84b70fe6-c76d-64db-55d2-6d8ba56b05a0 [Follower]" leader= +testLoggerAdapter verbose: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.052Z [INFO] server-22547879-cf85-d863-3037-0bc987d590a5: entering follower state: follower="Node at 22547879-cf85-d863-3037-0bc987d590a5 [Follower]" leader= + testing.go:149: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.052Z [INFO] server-22547879-cf85-d863-3037-0bc987d590a5: entering follower state: follower="Node at 22547879-cf85-d863-3037-0bc987d590a5 [Follower]" leader= +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.053Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering follower state: follower="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Follower]" leader= + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.053Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering follower state: follower="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Follower]" leader= +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.118Z [WARN] server-f9d19753-8b02-b0b4-8691-a94389c53005: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.118Z [WARN] server-f9d19753-8b02-b0b4-8691-a94389c53005: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering candidate state: node="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Candidate]" term=2 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering candidate state: node="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Candidate]" term=2 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: election won: tally=2 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: election won: tally=2 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering leader state: leader="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Leader]" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: entering leader state: leader="Node at f9d19753-8b02-b0b4-8691-a94389c53005 [Leader]" +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: added peer, starting replication: peer=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.119Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: added peer, starting replication: peer=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.120Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: added peer, starting replication: peer=server-22547879-cf85-d863-3037-0bc987d590a5 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.120Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: added peer, starting replication: peer=server-22547879-cf85-d863-3037-0bc987d590a5 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.120Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: pipelining replication: peer="{Voter server-22547879-cf85-d863-3037-0bc987d590a5 22547879-cf85-d863-3037-0bc987d590a5}" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.120Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: pipelining replication: peer="{Voter server-22547879-cf85-d863-3037-0bc987d590a5 22547879-cf85-d863-3037-0bc987d590a5}" +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.121Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: pipelining replication: peer="{Voter server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 84b70fe6-c76d-64db-55d2-6d8ba56b05a0}" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.121Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: pipelining replication: peer="{Voter server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 84b70fe6-c76d-64db-55d2-6d8ba56b05a0}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.229Z [INFO] stable state for Leader reached at 2024-01-08 10:38:00.12153303 +0000 UTC m=+8.273327200 (1 nodes), 68.87672ms from start of poll, 71.134863ms from cluster start. Timeout at 2024-01-08 10:38:00.229050753 +0000 UTC m=+8.380844879, 107.517679ms after stability + testing.go:149: cluster: 2024-01-08T10:38:00.229Z [INFO] stable state for Leader reached at 2024-01-08 10:38:00.12153303 +0000 UTC m=+8.273327200 (1 nodes), 68.87672ms from start of poll, 71.134863ms from cluster start. Timeout at 2024-01-08 10:38:00.229050753 +0000 UTC m=+8.380844879, 107.517679ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.234Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:00.234Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.342Z [INFO] stable state for Follower reached at 2024-01-08 10:38:00.234951916 +0000 UTC m=+8.386746030 (2 nodes), 4.984µs from start of poll, 184.553693ms from cluster start. Timeout at 2024-01-08 10:38:00.342022669 +0000 UTC m=+8.493816803, 107.070773ms after stability + testing.go:149: cluster: 2024-01-08T10:38:00.342Z [INFO] stable state for Follower reached at 2024-01-08 10:38:00.234951916 +0000 UTC m=+8.386746030 (2 nodes), 4.984µs from start of poll, 184.553693ms from cluster start. Timeout at 2024-01-08 10:38:00.342022669 +0000 UTC m=+8.493816803, 107.070773ms after stability +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.342Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: updating configuration: command=RemoveServer server-id=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 server-addr= servers="[{Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.342Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: updating configuration: command=RemoveServer server-id=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 server-addr= servers="[{Suffrage:Voter ID:server-22547879-cf85-d863-3037-0bc987d590a5 Address:22547879-cf85-d863-3037-0bc987d590a5} {Suffrage:Voter ID:server-f9d19753-8b02-b0b4-8691-a94389c53005 Address:f9d19753-8b02-b0b4-8691-a94389c53005}]" +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.342Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: removed peer, stopping replication: peer=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 last-index=3 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.342Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: removed peer, stopping replication: peer=server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 last-index=3 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.343Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: aborting pipeline replication: peer="{Voter server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 84b70fe6-c76d-64db-55d2-6d8ba56b05a0}" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.343Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: aborting pipeline replication: peer="{Voter server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0 84b70fe6-c76d-64db-55d2-6d8ba56b05a0}" +testLoggerAdapter verbose: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.438Z [WARN] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: heartbeat timeout reached, starting election: last-leader=f9d19753-8b02-b0b4-8691-a94389c53005 + testing.go:149: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.438Z [WARN] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: heartbeat timeout reached, starting election: last-leader=f9d19753-8b02-b0b4-8691-a94389c53005 +testLoggerAdapter verbose: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.438Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: entering candidate state: node="Node at 84b70fe6-c76d-64db-55d2-6d8ba56b05a0 [Candidate]" term=3 + testing.go:149: server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: 2024-01-08T10:38:00.438Z [INFO] server-84b70fe6-c76d-64db-55d2-6d8ba56b05a0: entering candidate state: node="Node at 84b70fe6-c76d-64db-55d2-6d8ba56b05a0 [Candidate]" term=3 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.438Z [WARN] server-f9d19753-8b02-b0b4-8691-a94389c53005: rejecting vote request since we have a leader: from=84b70fe6-c76d-64db-55d2-6d8ba56b05a0 leader=f9d19753-8b02-b0b4-8691-a94389c53005 + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.438Z [WARN] server-f9d19753-8b02-b0b4-8691-a94389c53005: rejecting vote request since we have a leader: from=84b70fe6-c76d-64db-55d2-6d8ba56b05a0 leader=f9d19753-8b02-b0b4-8691-a94389c53005 +testLoggerAdapter verbose: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.439Z [WARN] server-22547879-cf85-d863-3037-0bc987d590a5: rejecting vote request since we have a leader: from=84b70fe6-c76d-64db-55d2-6d8ba56b05a0 leader=f9d19753-8b02-b0b4-8691-a94389c53005 + testing.go:149: server-22547879-cf85-d863-3037-0bc987d590a5: 2024-01-08T10:38:00.439Z [WARN] server-22547879-cf85-d863-3037-0bc987d590a5: rejecting vote request since we have a leader: from=84b70fe6-c76d-64db-55d2-6d8ba56b05a0 leader=f9d19753-8b02-b0b4-8691-a94389c53005 +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.450Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: aborting pipeline replication: peer="{Voter server-22547879-cf85-d863-3037-0bc987d590a5 22547879-cf85-d863-3037-0bc987d590a5}" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.450Z [INFO] server-f9d19753-8b02-b0b4-8691-a94389c53005: aborting pipeline replication: peer="{Voter server-22547879-cf85-d863-3037-0bc987d590a5 22547879-cf85-d863-3037-0bc987d590a5}" +testLoggerAdapter verbose: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.505Z [ERROR] server-f9d19753-8b02-b0b4-8691-a94389c53005: failed to heartbeat to: peer=22547879-cf85-d863-3037-0bc987d590a5 error="command timed out" + testing.go:149: server-f9d19753-8b02-b0b4-8691-a94389c53005: 2024-01-08T10:38:00.505Z [ERROR] server-f9d19753-8b02-b0b4-8691-a94389c53005: failed to heartbeat to: peer=22547879-cf85-d863-3037-0bc987d590a5 error="command timed out" +--- PASS: TestRaft_RemoveFollower (0.46s) === RUN TestRaft_RemoveLeader -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.645Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.645Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.645Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.645Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.645Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.645Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-25624f96-c9b5-cd5e-601a-eb7e239f288a Address:25624f96-c9b5-cd5e-601a-eb7e239f288a} {Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.645Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:13.645Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.645Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering follower state: follower="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Follower]" leader= - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.645Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering follower state: follower="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Follower]" leader= -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.645Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering follower state: follower="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Follower]" leader= - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.645Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering follower state: follower="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Follower]" leader= -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.645Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader= - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.645Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader= -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [WARN] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [WARN] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering candidate state: node="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Candidate]" term=2 - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering candidate state: node="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Candidate]" term=2 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering candidate state: node="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Candidate]" term=2 - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering candidate state: node="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Candidate]" term=2 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: duplicate requestVote for same term: term=2 - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.702Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: election won: tally=2 - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: election won: tally=2 -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering leader state: leader="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Leader]" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: entering leader state: leader="Node at 25624f96-c9b5-cd5e-601a-eb7e239f288a [Leader]" -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: added peer, starting replication: peer=server-e302a44c-9970-c5d6-a626-74e3f3d554e2 - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: added peer, starting replication: peer=server-e302a44c-9970-c5d6-a626-74e3f3d554e2 -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: added peer, starting replication: peer=server-36cfb538-214d-4985-fd24-19cfc744b5d8 - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.702Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: added peer, starting replication: peer=server-36cfb538-214d-4985-fd24-19cfc744b5d8 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.703Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.703Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: pipelining replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: pipelining replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.703Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: duplicate requestVote for same term: term=2 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.703Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: duplicate requestVote for same term: term=2 -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: pipelining replication: peer="{Voter server-e302a44c-9970-c5d6-a626-74e3f3d554e2 e302a44c-9970-c5d6-a626-74e3f3d554e2}" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: pipelining replication: peer="{Voter server-e302a44c-9970-c5d6-a626-74e3f3d554e2 e302a44c-9970-c5d6-a626-74e3f3d554e2}" -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [WARN] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: rejecting vote request since we have a leader: from=36cfb538-214d-4985-fd24-19cfc744b5d8 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.703Z [WARN] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: rejecting vote request since we have a leader: from=36cfb538-214d-4985-fd24-19cfc744b5d8 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.808Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.703262206 +0000 UTC m=+8.033494894 (1 nodes), 57.459445ms from start of poll, 57.875472ms from cluster start. Timeout at 2025-02-09 16:52:13.808901504 +0000 UTC m=+8.139134192, 105.639298ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.808Z [INFO] stable state for Leader reached at 2025-02-09 16:52:13.703262206 +0000 UTC m=+8.033494894 (1 nodes), 57.459445ms from start of poll, 57.875472ms from cluster start. Timeout at 2025-02-09 16:52:13.808901504 +0000 UTC m=+8.139134192, 105.639298ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.814Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:13.814Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:13.920Z [INFO] stable state for Follower reached at 2025-02-09 16:52:13.814621126 +0000 UTC m=+8.144853812 (2 nodes), 994ns from start of poll, 169.23439ms from cluster start. Timeout at 2025-02-09 16:52:13.920272331 +0000 UTC m=+8.250505019, 105.651207ms after stability - testing.go:149: cluster: 2025-02-09T16:52:13.920Z [INFO] stable state for Follower reached at 2025-02-09 16:52:13.814621126 +0000 UTC m=+8.144853812 (2 nodes), 994ns from start of poll, 169.23439ms from cluster start. Timeout at 2025-02-09 16:52:13.920272331 +0000 UTC m=+8.250505019, 105.651207ms after stability -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: updating configuration: command=RemoveServer server-id=server-25624f96-c9b5-cd5e-601a-eb7e239f288a server-addr= servers="[{Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: updating configuration: command=RemoveServer server-id=server-25624f96-c9b5-cd5e-601a-eb7e239f288a server-addr= servers="[{Suffrage:Voter ID:server-e302a44c-9970-c5d6-a626-74e3f3d554e2 Address:e302a44c-9970-c5d6-a626-74e3f3d554e2} {Suffrage:Voter ID:server-36cfb538-214d-4985-fd24-19cfc744b5d8 Address:36cfb538-214d-4985-fd24-19cfc744b5d8}]" -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: removed ourself, shutting down - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: removed ourself, shutting down -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: aborting pipeline replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: aborting pipeline replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" -testLoggerAdapter verbose: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: aborting pipeline replication: peer="{Voter server-e302a44c-9970-c5d6-a626-74e3f3d554e2 e302a44c-9970-c5d6-a626-74e3f3d554e2}" - testing.go:149: server-25624f96-c9b5-cd5e-601a-eb7e239f288a: 2025-02-09T16:52:13.920Z [INFO] server-25624f96-c9b5-cd5e-601a-eb7e239f288a: aborting pipeline replication: peer="{Voter server-e302a44c-9970-c5d6-a626-74e3f3d554e2 e302a44c-9970-c5d6-a626-74e3f3d554e2}" -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.975Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: heartbeat timeout reached, starting election: last-leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.975Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: heartbeat timeout reached, starting election: last-leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.975Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=3 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:13.975Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=3 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.975Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: rejecting vote request since we have a leader: from=e302a44c-9970-c5d6-a626-74e3f3d554e2 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:13.975Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: rejecting vote request since we have a leader: from=e302a44c-9970-c5d6-a626-74e3f3d554e2 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.033Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: Election timeout reached, restarting election - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.033Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: Election timeout reached, restarting election -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.033Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=4 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.033Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=4 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.033Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: rejecting vote request since we have a leader: from=e302a44c-9970-c5d6-a626-74e3f3d554e2 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.033Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: rejecting vote request since we have a leader: from=e302a44c-9970-c5d6-a626-74e3f3d554e2 leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: heartbeat timeout reached, starting election: last-leader=25624f96-c9b5-cd5e-601a-eb7e239f288a - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [WARN] server-36cfb538-214d-4985-fd24-19cfc744b5d8: heartbeat timeout reached, starting election: last-leader=25624f96-c9b5-cd5e-601a-eb7e239f288a -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering candidate state: node="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Candidate]" term=3 - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering candidate state: node="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Candidate]" term=3 -testLoggerAdapter verbose: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader= - testing.go:149: server-36cfb538-214d-4985-fd24-19cfc744b5d8: 2025-02-09T16:52:14.034Z [INFO] server-36cfb538-214d-4985-fd24-19cfc744b5d8: entering follower state: follower="Node at 36cfb538-214d-4985-fd24-19cfc744b5d8 [Follower]" leader= -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: Election timeout reached, restarting election - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [WARN] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: Election timeout reached, restarting election -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=5 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering candidate state: node="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Candidate]" term=5 -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: election won: tally=2 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: election won: tally=2 -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering leader state: leader="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Leader]" - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: entering leader state: leader="Node at e302a44c-9970-c5d6-a626-74e3f3d554e2 [Leader]" -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: added peer, starting replication: peer=server-36cfb538-214d-4985-fd24-19cfc744b5d8 - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: added peer, starting replication: peer=server-36cfb538-214d-4985-fd24-19cfc744b5d8 -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: pipelining replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.087Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: pipelining replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.131Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:14.131Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.237Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.131539506 +0000 UTC m=+8.461772209 (1 nodes), 34.405µs from start of poll, 486.152787ms from cluster start. Timeout at 2025-02-09 16:52:14.237360643 +0000 UTC m=+8.567593331, 105.821122ms after stability - testing.go:149: cluster: 2025-02-09T16:52:14.237Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.131539506 +0000 UTC m=+8.461772209 (1 nodes), 34.405µs from start of poll, 486.152787ms from cluster start. Timeout at 2025-02-09 16:52:14.237360643 +0000 UTC m=+8.567593331, 105.821122ms after stability -testLoggerAdapter verbose: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.237Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: aborting pipeline replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" - testing.go:149: server-e302a44c-9970-c5d6-a626-74e3f3d554e2: 2025-02-09T16:52:14.237Z [INFO] server-e302a44c-9970-c5d6-a626-74e3f3d554e2: aborting pipeline replication: peer="{Voter server-36cfb538-214d-4985-fd24-19cfc744b5d8 36cfb538-214d-4985-fd24-19cfc744b5d8}" ---- PASS: TestRaft_RemoveLeader (0.59s) +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.510Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.510Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.511Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.511Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.512Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.512Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-5975690d-1868-8a83-6ce3-4ae1c6ed04de Address:5975690d-1868-8a83-6ce3-4ae1c6ed04de} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.512Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:00.512Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.512Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering follower state: follower="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Follower]" leader= + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.512Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering follower state: follower="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Follower]" leader= +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.512Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering follower state: follower="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Follower]" leader= + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.512Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering follower state: follower="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Follower]" leader= +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.513Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering follower state: follower="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Follower]" leader= + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.513Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering follower state: follower="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Follower]" leader= +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [WARN] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [WARN] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering candidate state: node="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Candidate]" term=2 + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering candidate state: node="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Candidate]" term=2 +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: election won: tally=2 + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.582Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: election won: tally=2 +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering leader state: leader="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Leader]" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: entering leader state: leader="Node at 5975690d-1868-8a83-6ce3-4ae1c6ed04de [Leader]" +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: added peer, starting replication: peer=server-d83d07be-447f-0bc8-4019-bcb5a108c89b + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: added peer, starting replication: peer=server-d83d07be-447f-0bc8-4019-bcb5a108c89b +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: added peer, starting replication: peer=server-7a048273-947a-e9e8-e289-44a30dd899ff + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: added peer, starting replication: peer=server-7a048273-947a-e9e8-e289-44a30dd899ff +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: pipelining replication: peer="{Voter server-7a048273-947a-e9e8-e289-44a30dd899ff 7a048273-947a-e9e8-e289-44a30dd899ff}" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.583Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: pipelining replication: peer="{Voter server-7a048273-947a-e9e8-e289-44a30dd899ff 7a048273-947a-e9e8-e289-44a30dd899ff}" +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.584Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: pipelining replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.584Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: pipelining replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.694Z [INFO] stable state for Leader reached at 2024-01-08 10:38:00.585196963 +0000 UTC m=+8.736991136 (1 nodes), 72.617444ms from start of poll, 75.967886ms from cluster start. Timeout at 2024-01-08 10:38:00.694087325 +0000 UTC m=+8.845881486, 108.89035ms after stability + testing.go:149: cluster: 2024-01-08T10:38:00.694Z [INFO] stable state for Leader reached at 2024-01-08 10:38:00.585196963 +0000 UTC m=+8.736991136 (1 nodes), 72.617444ms from start of poll, 75.967886ms from cluster start. Timeout at 2024-01-08 10:38:00.694087325 +0000 UTC m=+8.845881486, 108.89035ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.701Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:00.701Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:00.810Z [INFO] stable state for Follower reached at 2024-01-08 10:38:00.70229509 +0000 UTC m=+8.854089216 (2 nodes), 5.042µs from start of poll, 193.065966ms from cluster start. Timeout at 2024-01-08 10:38:00.809962127 +0000 UTC m=+8.961756365, 107.667149ms after stability + testing.go:149: cluster: 2024-01-08T10:38:00.810Z [INFO] stable state for Follower reached at 2024-01-08 10:38:00.70229509 +0000 UTC m=+8.854089216 (2 nodes), 5.042µs from start of poll, 193.065966ms from cluster start. Timeout at 2024-01-08 10:38:00.809962127 +0000 UTC m=+8.961756365, 107.667149ms after stability +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.810Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: updating configuration: command=RemoveServer server-id=server-5975690d-1868-8a83-6ce3-4ae1c6ed04de server-addr= servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.810Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: updating configuration: command=RemoveServer server-id=server-5975690d-1868-8a83-6ce3-4ae1c6ed04de server-addr= servers="[{Suffrage:Voter ID:server-d83d07be-447f-0bc8-4019-bcb5a108c89b Address:d83d07be-447f-0bc8-4019-bcb5a108c89b} {Suffrage:Voter ID:server-7a048273-947a-e9e8-e289-44a30dd899ff Address:7a048273-947a-e9e8-e289-44a30dd899ff}]" +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: removed ourself, shutting down + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: removed ourself, shutting down +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: aborting pipeline replication: peer="{Voter server-7a048273-947a-e9e8-e289-44a30dd899ff 7a048273-947a-e9e8-e289-44a30dd899ff}" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: aborting pipeline replication: peer="{Voter server-7a048273-947a-e9e8-e289-44a30dd899ff 7a048273-947a-e9e8-e289-44a30dd899ff}" +testLoggerAdapter verbose: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: aborting pipeline replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" + testing.go:149: server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: 2024-01-08T10:38:00.811Z [INFO] server-5975690d-1868-8a83-6ce3-4ae1c6ed04de: aborting pipeline replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.914Z [WARN] server-7a048273-947a-e9e8-e289-44a30dd899ff: heartbeat timeout reached, starting election: last-leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.914Z [WARN] server-7a048273-947a-e9e8-e289-44a30dd899ff: heartbeat timeout reached, starting election: last-leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.914Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering candidate state: node="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Candidate]" term=3 + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.914Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering candidate state: node="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Candidate]" term=3 +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.914Z [WARN] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: rejecting vote request since we have a leader: from=7a048273-947a-e9e8-e289-44a30dd899ff leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.914Z [WARN] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: rejecting vote request since we have a leader: from=7a048273-947a-e9e8-e289-44a30dd899ff leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.934Z [WARN] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: heartbeat timeout reached, starting election: last-leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.934Z [WARN] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: heartbeat timeout reached, starting election: last-leader=5975690d-1868-8a83-6ce3-4ae1c6ed04de +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.934Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering candidate state: node="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Candidate]" term=3 + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:00.934Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering candidate state: node="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Candidate]" term=3 +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.934Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: duplicate requestVote for same term: term=3 + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:00.934Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.006Z [WARN] server-7a048273-947a-e9e8-e289-44a30dd899ff: Election timeout reached, restarting election + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.006Z [WARN] server-7a048273-947a-e9e8-e289-44a30dd899ff: Election timeout reached, restarting election +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.006Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering candidate state: node="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Candidate]" term=4 + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.006Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering candidate state: node="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Candidate]" term=4 +testLoggerAdapter verbose: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:01.006Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering follower state: follower="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Follower]" leader= + testing.go:149: server-d83d07be-447f-0bc8-4019-bcb5a108c89b: 2024-01-08T10:38:01.006Z [INFO] server-d83d07be-447f-0bc8-4019-bcb5a108c89b: entering follower state: follower="Node at d83d07be-447f-0bc8-4019-bcb5a108c89b [Follower]" leader= +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: election won: tally=2 + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: election won: tally=2 +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering leader state: leader="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Leader]" + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: entering leader state: leader="Node at 7a048273-947a-e9e8-e289-44a30dd899ff [Leader]" +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: added peer, starting replication: peer=server-d83d07be-447f-0bc8-4019-bcb5a108c89b + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.007Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: added peer, starting replication: peer=server-d83d07be-447f-0bc8-4019-bcb5a108c89b +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.008Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: pipelining replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.008Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: pipelining replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.025Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:01.025Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.138Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.025947315 +0000 UTC m=+9.177741167 (1 nodes), 43.009µs from start of poll, 516.717917ms from cluster start. Timeout at 2024-01-08 10:38:01.138269052 +0000 UTC m=+9.290063193, 112.322026ms after stability + testing.go:149: cluster: 2024-01-08T10:38:01.138Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.025947315 +0000 UTC m=+9.177741167 (1 nodes), 43.009µs from start of poll, 516.717917ms from cluster start. Timeout at 2024-01-08 10:38:01.138269052 +0000 UTC m=+9.290063193, 112.322026ms after stability +testLoggerAdapter verbose: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.138Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: aborting pipeline replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" + testing.go:149: server-7a048273-947a-e9e8-e289-44a30dd899ff: 2024-01-08T10:38:01.138Z [INFO] server-7a048273-947a-e9e8-e289-44a30dd899ff: aborting pipeline replication: peer="{Voter server-d83d07be-447f-0bc8-4019-bcb5a108c89b d83d07be-447f-0bc8-4019-bcb5a108c89b}" +--- PASS: TestRaft_RemoveLeader (0.63s) === RUN TestRaft_RemoveLeader_NoShutdown -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.238Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.238Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.238Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.238Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.238Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.238Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851} {Suffrage:Voter ID:server-3b3cecc4-6229-20db-0d25-2574566cc514 Address:3b3cecc4-6229-20db-0d25-2574566cc514}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.238Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:14.238Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.238Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering follower state: follower="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Follower]" leader= - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.238Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering follower state: follower="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Follower]" leader= -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.238Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering follower state: follower="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Follower]" leader= - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.238Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering follower state: follower="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Follower]" leader= -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.238Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering follower state: follower="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Follower]" leader= - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.238Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering follower state: follower="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Follower]" leader= -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [WARN] server-3b3cecc4-6229-20db-0d25-2574566cc514: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [WARN] server-3b3cecc4-6229-20db-0d25-2574566cc514: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering candidate state: node="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Candidate]" term=2 - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering candidate state: node="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Candidate]" term=2 -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: election won: tally=2 - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: election won: tally=2 -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering leader state: leader="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Leader]" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering leader state: leader="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Leader]" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: added peer, starting replication: peer=server-921b87d6-5ba1-cc98-59c6-b2d8976d315a - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: added peer, starting replication: peer=server-921b87d6-5ba1-cc98-59c6-b2d8976d315a -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: added peer, starting replication: peer=server-09107ffa-a86a-efad-333b-a373ce69d851 - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: added peer, starting replication: peer=server-09107ffa-a86a-efad-333b-a373ce69d851 -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: pipelining replication: peer="{Voter server-09107ffa-a86a-efad-333b-a373ce69d851 09107ffa-a86a-efad-333b-a373ce69d851}" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: pipelining replication: peer="{Voter server-09107ffa-a86a-efad-333b-a373ce69d851 09107ffa-a86a-efad-333b-a373ce69d851}" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: pipelining replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.298Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: pipelining replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.404Z [INFO] stable state for Follower reached at 2025-02-09 16:52:14.299047851 +0000 UTC m=+8.629280539 (2 nodes), 60.539256ms from start of poll, 60.933911ms from cluster start. Timeout at 2025-02-09 16:52:14.404350543 +0000 UTC m=+8.734583229, 105.30269ms after stability - testing.go:149: cluster: 2025-02-09T16:52:14.404Z [INFO] stable state for Follower reached at 2025-02-09 16:52:14.299047851 +0000 UTC m=+8.629280539 (2 nodes), 60.539256ms from start of poll, 60.933911ms from cluster start. Timeout at 2025-02-09 16:52:14.404350543 +0000 UTC m=+8.734583229, 105.30269ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.404Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:14.404Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.510Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.404486871 +0000 UTC m=+8.734719557 (1 nodes), 1.234µs from start of poll, 166.372929ms from cluster start. Timeout at 2025-02-09 16:52:14.510232476 +0000 UTC m=+8.840465162, 105.745605ms after stability - testing.go:149: cluster: 2025-02-09T16:52:14.510Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.404486871 +0000 UTC m=+8.734719557 (1 nodes), 1.234µs from start of poll, 166.372929ms from cluster start. Timeout at 2025-02-09 16:52:14.510232476 +0000 UTC m=+8.840465162, 105.745605ms after stability -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.514Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: updating configuration: command=RemoveServer server-id=server-3b3cecc4-6229-20db-0d25-2574566cc514 server-addr= servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851}]" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.514Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: updating configuration: command=RemoveServer server-id=server-3b3cecc4-6229-20db-0d25-2574566cc514 server-addr= servers="[{Suffrage:Voter ID:server-921b87d6-5ba1-cc98-59c6-b2d8976d315a Address:921b87d6-5ba1-cc98-59c6-b2d8976d315a} {Suffrage:Voter ID:server-09107ffa-a86a-efad-333b-a373ce69d851 Address:09107ffa-a86a-efad-333b-a373ce69d851}]" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.515Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: removed ourself, transitioning to follower - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.515Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: removed ourself, transitioning to follower -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering follower state: follower="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Follower]" leader= - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: entering follower state: follower="Node at 3b3cecc4-6229-20db-0d25-2574566cc514 [Follower]" leader= -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: aborting pipeline replication: peer="{Voter server-09107ffa-a86a-efad-333b-a373ce69d851 09107ffa-a86a-efad-333b-a373ce69d851}" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: aborting pipeline replication: peer="{Voter server-09107ffa-a86a-efad-333b-a373ce69d851 09107ffa-a86a-efad-333b-a373ce69d851}" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: aborting pipeline replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.516Z [INFO] server-3b3cecc4-6229-20db-0d25-2574566cc514: aborting pipeline replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" -testLoggerAdapter verbose: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.586Z [WARN] server-3b3cecc4-6229-20db-0d25-2574566cc514: not part of stable configuration, aborting election - testing.go:149: server-3b3cecc4-6229-20db-0d25-2574566cc514: 2025-02-09T16:52:14.586Z [WARN] server-3b3cecc4-6229-20db-0d25-2574566cc514: not part of stable configuration, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.622Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:14.622Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.627Z [WARN] server-09107ffa-a86a-efad-333b-a373ce69d851: heartbeat timeout reached, starting election: last-leader=3b3cecc4-6229-20db-0d25-2574566cc514 - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.627Z [WARN] server-09107ffa-a86a-efad-333b-a373ce69d851: heartbeat timeout reached, starting election: last-leader=3b3cecc4-6229-20db-0d25-2574566cc514 -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.627Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering candidate state: node="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Candidate]" term=3 - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.627Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering candidate state: node="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Candidate]" term=3 -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.627Z [WARN] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: rejecting vote request since we have a leader: from=09107ffa-a86a-efad-333b-a373ce69d851 leader=3b3cecc4-6229-20db-0d25-2574566cc514 - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.627Z [WARN] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: rejecting vote request since we have a leader: from=09107ffa-a86a-efad-333b-a373ce69d851 leader=3b3cecc4-6229-20db-0d25-2574566cc514 -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.645Z [WARN] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: heartbeat timeout reached, starting election: last-leader=3b3cecc4-6229-20db-0d25-2574566cc514 - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.645Z [WARN] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: heartbeat timeout reached, starting election: last-leader=3b3cecc4-6229-20db-0d25-2574566cc514 -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.645Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering candidate state: node="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Candidate]" term=3 - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.645Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering candidate state: node="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Candidate]" term=3 -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.645Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: duplicate requestVote for same term: term=3 - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.645Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: duplicate requestVote for same term: term=3 -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [WARN] server-09107ffa-a86a-efad-333b-a373ce69d851: Election timeout reached, restarting election - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [WARN] server-09107ffa-a86a-efad-333b-a373ce69d851: Election timeout reached, restarting election -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering candidate state: node="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Candidate]" term=4 - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering candidate state: node="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Candidate]" term=4 -testLoggerAdapter verbose: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.714Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering follower state: follower="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Follower]" leader= - testing.go:149: server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: 2025-02-09T16:52:14.714Z [INFO] server-921b87d6-5ba1-cc98-59c6-b2d8976d315a: entering follower state: follower="Node at 921b87d6-5ba1-cc98-59c6-b2d8976d315a [Follower]" leader= -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: election won: tally=2 - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: election won: tally=2 -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering leader state: leader="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Leader]" - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: entering leader state: leader="Node at 09107ffa-a86a-efad-333b-a373ce69d851 [Leader]" -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: added peer, starting replication: peer=server-921b87d6-5ba1-cc98-59c6-b2d8976d315a - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.714Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: added peer, starting replication: peer=server-921b87d6-5ba1-cc98-59c6-b2d8976d315a -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.715Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: pipelining replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.715Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: pipelining replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.821Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.715126831 +0000 UTC m=+9.045359519 (1 nodes), 92.713542ms from start of poll, 477.012891ms from cluster start. Timeout at 2025-02-09 16:52:14.821153953 +0000 UTC m=+9.151386639, 106.02712ms after stability - testing.go:149: cluster: 2025-02-09T16:52:14.821Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.715126831 +0000 UTC m=+9.045359519 (1 nodes), 92.713542ms from start of poll, 477.012891ms from cluster start. Timeout at 2025-02-09 16:52:14.821153953 +0000 UTC m=+9.151386639, 106.02712ms after stability -testLoggerAdapter verbose: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.927Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: aborting pipeline replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" - testing.go:149: server-09107ffa-a86a-efad-333b-a373ce69d851: 2025-02-09T16:52:14.927Z [INFO] server-09107ffa-a86a-efad-333b-a373ce69d851: aborting pipeline replication: peer="{Voter server-921b87d6-5ba1-cc98-59c6-b2d8976d315a 921b87d6-5ba1-cc98-59c6-b2d8976d315a}" ---- PASS: TestRaft_RemoveLeader_NoShutdown (0.69s) +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.143Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.143Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.144Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.144Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.145Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.145Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering follower state: follower="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Follower]" leader= +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.146Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering follower state: follower="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Follower]" leader= + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.145Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-230b57ef-cd8a-8661-7c26-df088e513909 Address:230b57ef-cd8a-8661-7c26-df088e513909} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.148Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:01.148Z [INFO] starting stability test: raft-state=Follower + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.145Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering follower state: follower="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Follower]" leader= + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.146Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering follower state: follower="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Follower]" leader= +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.149Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader= + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.149Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader= +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.209Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.209Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering candidate state: node="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Candidate]" term=2 + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering candidate state: node="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Candidate]" term=2 +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: election won: tally=2 + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: election won: tally=2 +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering leader state: leader="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Leader]" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering leader state: leader="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Leader]" +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: added peer, starting replication: peer=server-ebf30ee7-202e-5f99-3085-d984e07eb50e +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.210Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: added peer, starting replication: peer=server-ebf30ee7-202e-5f99-3085-d984e07eb50e +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: added peer, starting replication: peer=server-a374c972-55fc-f15c-3680-3b7df70966f9 + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.210Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: added peer, starting replication: peer=server-a374c972-55fc-f15c-3680-3b7df70966f9 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.210Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering candidate state: node="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Candidate]" term=2 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering candidate state: node="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Candidate]" term=2 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: duplicate requestVote for same term: term=2 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.211Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.211Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: pipelining replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.211Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: pipelining replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.212Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: pipelining replication: peer="{Voter server-ebf30ee7-202e-5f99-3085-d984e07eb50e ebf30ee7-202e-5f99-3085-d984e07eb50e}" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.212Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: pipelining replication: peer="{Voter server-ebf30ee7-202e-5f99-3085-d984e07eb50e ebf30ee7-202e-5f99-3085-d984e07eb50e}" +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.212Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: rejecting vote request since we have a leader: from=a374c972-55fc-f15c-3680-3b7df70966f9 leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.212Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: rejecting vote request since we have a leader: from=a374c972-55fc-f15c-3680-3b7df70966f9 leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.213Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: rejecting vote request since we have a leader: from=a374c972-55fc-f15c-3680-3b7df70966f9 leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.213Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: rejecting vote request since we have a leader: from=a374c972-55fc-f15c-3680-3b7df70966f9 leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.318Z [INFO] stable state for Follower reached at 2024-01-08 10:38:01.213557868 +0000 UTC m=+9.365352050 (2 nodes), 64.415558ms from start of poll, 70.419254ms from cluster start. Timeout at 2024-01-08 10:38:01.318673161 +0000 UTC m=+9.470467013, 105.114963ms after stability + testing.go:149: cluster: 2024-01-08T10:38:01.318Z [INFO] stable state for Follower reached at 2024-01-08 10:38:01.213557868 +0000 UTC m=+9.365352050 (2 nodes), 64.415558ms from start of poll, 70.419254ms from cluster start. Timeout at 2024-01-08 10:38:01.318673161 +0000 UTC m=+9.470467013, 105.114963ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.318Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:01.318Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.425Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.318907449 +0000 UTC m=+9.470701299 (1 nodes), 1.406µs from start of poll, 175.768503ms from cluster start. Timeout at 2024-01-08 10:38:01.425795872 +0000 UTC m=+9.577589724, 106.888425ms after stability + testing.go:149: cluster: 2024-01-08T10:38:01.425Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.318907449 +0000 UTC m=+9.470701299 (1 nodes), 1.406µs from start of poll, 175.768503ms from cluster start. Timeout at 2024-01-08 10:38:01.425795872 +0000 UTC m=+9.577589724, 106.888425ms after stability +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.440Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: updating configuration: command=RemoveServer server-id=server-230b57ef-cd8a-8661-7c26-df088e513909 server-addr= servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.440Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: updating configuration: command=RemoveServer server-id=server-230b57ef-cd8a-8661-7c26-df088e513909 server-addr= servers="[{Suffrage:Voter ID:server-ebf30ee7-202e-5f99-3085-d984e07eb50e Address:ebf30ee7-202e-5f99-3085-d984e07eb50e} {Suffrage:Voter ID:server-a374c972-55fc-f15c-3680-3b7df70966f9 Address:a374c972-55fc-f15c-3680-3b7df70966f9}]" +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: removed ourself, transitioning to follower + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: removed ourself, transitioning to follower +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering follower state: follower="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Follower]" leader= + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: entering follower state: follower="Node at 230b57ef-cd8a-8661-7c26-df088e513909 [Follower]" leader= +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: aborting pipeline replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: aborting pipeline replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: aborting pipeline replication: peer="{Voter server-ebf30ee7-202e-5f99-3085-d984e07eb50e ebf30ee7-202e-5f99-3085-d984e07eb50e}" + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.443Z [INFO] server-230b57ef-cd8a-8661-7c26-df088e513909: aborting pipeline replication: peer="{Voter server-ebf30ee7-202e-5f99-3085-d984e07eb50e ebf30ee7-202e-5f99-3085-d984e07eb50e}" +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.519Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: heartbeat timeout reached, starting election: last-leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.519Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: heartbeat timeout reached, starting election: last-leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.519Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering candidate state: node="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Candidate]" term=3 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.519Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering candidate state: node="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Candidate]" term=3 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.519Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: rejecting vote request since we have a leader: from=ebf30ee7-202e-5f99-3085-d984e07eb50e leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.519Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: rejecting vote request since we have a leader: from=ebf30ee7-202e-5f99-3085-d984e07eb50e leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.526Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: heartbeat timeout reached, starting election: last-leader=230b57ef-cd8a-8661-7c26-df088e513909 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.526Z [WARN] server-a374c972-55fc-f15c-3680-3b7df70966f9: heartbeat timeout reached, starting election: last-leader=230b57ef-cd8a-8661-7c26-df088e513909 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.527Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering candidate state: node="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Candidate]" term=3 + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.527Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering candidate state: node="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Candidate]" term=3 +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.527Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: duplicate requestVote for same term: term=3 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.527Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: duplicate requestVote for same term: term=3 +testLoggerAdapter verbose: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.544Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: not part of stable configuration, aborting election + testing.go:149: server-230b57ef-cd8a-8661-7c26-df088e513909: 2024-01-08T10:38:01.544Z [WARN] server-230b57ef-cd8a-8661-7c26-df088e513909: not part of stable configuration, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.551Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:01.551Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.574Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: Election timeout reached, restarting election + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.574Z [WARN] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: Election timeout reached, restarting election +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.574Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering candidate state: node="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Candidate]" term=4 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.574Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering candidate state: node="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Candidate]" term=4 +testLoggerAdapter verbose: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.574Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader= + testing.go:149: server-a374c972-55fc-f15c-3680-3b7df70966f9: 2024-01-08T10:38:01.574Z [INFO] server-a374c972-55fc-f15c-3680-3b7df70966f9: entering follower state: follower="Node at a374c972-55fc-f15c-3680-3b7df70966f9 [Follower]" leader= +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: election won: tally=2 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: election won: tally=2 +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering leader state: leader="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Leader]" + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: entering leader state: leader="Node at ebf30ee7-202e-5f99-3085-d984e07eb50e [Leader]" +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: added peer, starting replication: peer=server-a374c972-55fc-f15c-3680-3b7df70966f9 + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: added peer, starting replication: peer=server-a374c972-55fc-f15c-3680-3b7df70966f9 +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: pipelining replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.575Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: pipelining replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.682Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.576079745 +0000 UTC m=+9.727873882 (1 nodes), 24.669794ms from start of poll, 432.941086ms from cluster start. Timeout at 2024-01-08 10:38:01.681882083 +0000 UTC m=+9.833676222, 105.80234ms after stability + testing.go:149: cluster: 2024-01-08T10:38:01.682Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.576079745 +0000 UTC m=+9.727873882 (1 nodes), 24.669794ms from start of poll, 432.941086ms from cluster start. Timeout at 2024-01-08 10:38:01.681882083 +0000 UTC m=+9.833676222, 105.80234ms after stability +testLoggerAdapter verbose: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.790Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: aborting pipeline replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" + testing.go:149: server-ebf30ee7-202e-5f99-3085-d984e07eb50e: 2024-01-08T10:38:01.790Z [INFO] server-ebf30ee7-202e-5f99-3085-d984e07eb50e: aborting pipeline replication: peer="{Voter server-a374c972-55fc-f15c-3680-3b7df70966f9 a374c972-55fc-f15c-3680-3b7df70966f9}" +--- PASS: TestRaft_RemoveLeader_NoShutdown (0.65s) === RUN TestRaft_RemoveFollower_SplitCluster -testLoggerAdapter verbose: server-c3e38f57-e3c1-594f-6781-22113fe78d22: 2025-02-09T16:52:14.927Z [INFO] server-c3e38f57-e3c1-594f-6781-22113fe78d22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" - testing.go:149: server-c3e38f57-e3c1-594f-6781-22113fe78d22: 2025-02-09T16:52:14.927Z [INFO] server-c3e38f57-e3c1-594f-6781-22113fe78d22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" -testLoggerAdapter verbose: server-09770b7f-cd60-a691-7d9c-d702b0609b28: 2025-02-09T16:52:14.928Z [INFO] server-09770b7f-cd60-a691-7d9c-d702b0609b28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" - testing.go:149: server-09770b7f-cd60-a691-7d9c-d702b0609b28: 2025-02-09T16:52:14.928Z [INFO] server-09770b7f-cd60-a691-7d9c-d702b0609b28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.928Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.928Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" -testLoggerAdapter verbose: server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: 2025-02-09T16:52:14.928Z [INFO] server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" - testing.go:149: server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: 2025-02-09T16:52:14.928Z [INFO] server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371} {Suffrage:Voter ID:server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 Address:293b25f1-82fb-cd6d-7127-8cdaa60a4e22}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:14.928Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:14.928Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c3e38f57-e3c1-594f-6781-22113fe78d22: 2025-02-09T16:52:14.928Z [INFO] server-c3e38f57-e3c1-594f-6781-22113fe78d22: entering follower state: follower="Node at c3e38f57-e3c1-594f-6781-22113fe78d22 [Follower]" leader= - testing.go:149: server-c3e38f57-e3c1-594f-6781-22113fe78d22: 2025-02-09T16:52:14.928Z [INFO] server-c3e38f57-e3c1-594f-6781-22113fe78d22: entering follower state: follower="Node at c3e38f57-e3c1-594f-6781-22113fe78d22 [Follower]" leader= -testLoggerAdapter verbose: server-09770b7f-cd60-a691-7d9c-d702b0609b28: 2025-02-09T16:52:14.928Z [INFO] server-09770b7f-cd60-a691-7d9c-d702b0609b28: entering follower state: follower="Node at 09770b7f-cd60-a691-7d9c-d702b0609b28 [Follower]" leader= - testing.go:149: server-09770b7f-cd60-a691-7d9c-d702b0609b28: 2025-02-09T16:52:14.928Z [INFO] server-09770b7f-cd60-a691-7d9c-d702b0609b28: entering follower state: follower="Node at 09770b7f-cd60-a691-7d9c-d702b0609b28 [Follower]" leader= -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.928Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering follower state: follower="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Follower]" leader= - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.928Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering follower state: follower="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Follower]" leader= -testLoggerAdapter verbose: server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: 2025-02-09T16:52:14.928Z [INFO] server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: entering follower state: follower="Node at 293b25f1-82fb-cd6d-7127-8cdaa60a4e22 [Follower]" leader= - testing.go:149: server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: 2025-02-09T16:52:14.928Z [INFO] server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22: entering follower state: follower="Node at 293b25f1-82fb-cd6d-7127-8cdaa60a4e22 [Follower]" leader= -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering candidate state: node="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Candidate]" term=2 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering candidate state: node="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Candidate]" term=2 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: election won: tally=3 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: election won: tally=3 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering leader state: leader="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Leader]" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering leader state: leader="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Leader]" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-c3e38f57-e3c1-594f-6781-22113fe78d22 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-c3e38f57-e3c1-594f-6781-22113fe78d22 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-09770b7f-cd60-a691-7d9c-d702b0609b28 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-09770b7f-cd60-a691-7d9c-d702b0609b28 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: added peer, starting replication: peer=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 293b25f1-82fb-cd6d-7127-8cdaa60a4e22}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 293b25f1-82fb-cd6d-7127-8cdaa60a4e22}" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:14.998Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: pipelining replication: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.104Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.998455643 +0000 UTC m=+9.328688331 (1 nodes), 70.167026ms from start of poll, 70.583158ms from cluster start. Timeout at 2025-02-09 16:52:15.104491827 +0000 UTC m=+9.434724513, 106.036182ms after stability - testing.go:149: cluster: 2025-02-09T16:52:15.104Z [INFO] stable state for Leader reached at 2025-02-09 16:52:14.998455643 +0000 UTC m=+9.328688331 (1 nodes), 70.167026ms from start of poll, 70.583158ms from cluster start. Timeout at 2025-02-09 16:52:15.104491827 +0000 UTC m=+9.434724513, 106.036182ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.210Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:15.210Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.315Z [INFO] stable state for Follower reached at 2025-02-09 16:52:15.21017712 +0000 UTC m=+9.540409806 (3 nodes), 1.182µs from start of poll, 282.304633ms from cluster start. Timeout at 2025-02-09 16:52:15.315820469 +0000 UTC m=+9.646053156, 105.64335ms after stability - testing.go:149: cluster: 2025-02-09T16:52:15.315Z [INFO] stable state for Follower reached at 2025-02-09 16:52:15.21017712 +0000 UTC m=+9.540409806 (3 nodes), 1.182µs from start of poll, 282.304633ms from cluster start. Timeout at 2025-02-09 16:52:15.315820469 +0000 UTC m=+9.646053156, 105.64335ms after stability -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.315Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: updating configuration: command=RemoveServer server-id=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 server-addr= servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371}]" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.315Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: updating configuration: command=RemoveServer server-id=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 server-addr= servers="[{Suffrage:Voter ID:server-c3e38f57-e3c1-594f-6781-22113fe78d22 Address:c3e38f57-e3c1-594f-6781-22113fe78d22} {Suffrage:Voter ID:server-09770b7f-cd60-a691-7d9c-d702b0609b28 Address:09770b7f-cd60-a691-7d9c-d702b0609b28} {Suffrage:Voter ID:server-19e4abba-dfde-6448-4f48-bf7641c8f371 Address:19e4abba-dfde-6448-4f48-bf7641c8f371}]" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: removed peer, stopping replication: peer=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 last-index=3 - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: removed peer, stopping replication: peer=server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 last-index=3 -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to pipeline appendEntries: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="append pipeline closed" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to pipeline appendEntries: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="append pipeline closed" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to pipeline appendEntries: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="append pipeline closed" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to pipeline appendEntries: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="append pipeline closed" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 293b25f1-82fb-cd6d-7127-8cdaa60a4e22}" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: aborting pipeline replication: peer="{Voter server-293b25f1-82fb-cd6d-7127-8cdaa60a4e22 293b25f1-82fb-cd6d-7127-8cdaa60a4e22}" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.316Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.319Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.319Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.323Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.323Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.325Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.325Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.335Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.335Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.339Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.339Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.341Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.341Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.346Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.346Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.353Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.353Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=c3e38f57-e3c1-594f-6781-22113fe78d22 error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.358Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.358Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to heartbeat to: peer=09770b7f-cd60-a691-7d9c-d702b0609b28 error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.359Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-c3e38f57-e3c1-594f-6781-22113fe78d22 time=50.439684ms - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.359Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-c3e38f57-e3c1-594f-6781-22113fe78d22 time=50.439684ms -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.363Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.363Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-c3e38f57-e3c1-594f-6781-22113fe78d22 c3e38f57-e3c1-594f-6781-22113fe78d22}" error="failed to connect to peer: c3e38f57-e3c1-594f-6781-22113fe78d22" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.365Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.365Z [ERROR] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to appendEntries to: peer="{Voter server-09770b7f-cd60-a691-7d9c-d702b0609b28 09770b7f-cd60-a691-7d9c-d702b0609b28}" error="failed to connect to peer: 09770b7f-cd60-a691-7d9c-d702b0609b28" -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-c3e38f57-e3c1-594f-6781-22113fe78d22 time=61.396625ms - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-c3e38f57-e3c1-594f-6781-22113fe78d22 time=61.396625ms -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-09770b7f-cd60-a691-7d9c-d702b0609b28 time=55.747352ms - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact: server-id=server-09770b7f-cd60-a691-7d9c-d702b0609b28 time=55.747352ms -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact quorum of nodes, stepping down - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [WARN] server-19e4abba-dfde-6448-4f48-bf7641c8f371: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering follower state: follower="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Follower]" leader= - testing.go:149: server-19e4abba-dfde-6448-4f48-bf7641c8f371: 2025-02-09T16:52:15.370Z [INFO] server-19e4abba-dfde-6448-4f48-bf7641c8f371: entering follower state: follower="Node at 19e4abba-dfde-6448-4f48-bf7641c8f371 [Follower]" leader= ---- PASS: TestRaft_RemoveFollower_SplitCluster (0.44s) +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.794Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.794Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" +testLoggerAdapter verbose: server-2690d475-d2db-301d-8d3e-5ec651a9bf47: 2024-01-08T10:38:01.795Z [INFO] server-2690d475-d2db-301d-8d3e-5ec651a9bf47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" + testing.go:149: server-2690d475-d2db-301d-8d3e-5ec651a9bf47: 2024-01-08T10:38:01.795Z [INFO] server-2690d475-d2db-301d-8d3e-5ec651a9bf47: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" +testLoggerAdapter verbose: server-21169459-7460-dcbb-5b04-fb17e5a32d36: 2024-01-08T10:38:01.796Z [INFO] server-21169459-7460-dcbb-5b04-fb17e5a32d36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" + testing.go:149: server-21169459-7460-dcbb-5b04-fb17e5a32d36: 2024-01-08T10:38:01.796Z [INFO] server-21169459-7460-dcbb-5b04-fb17e5a32d36: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" +testLoggerAdapter verbose: server-a4489edd-597d-3127-6e12-5f29081acf2f: 2024-01-08T10:38:01.797Z [INFO] server-a4489edd-597d-3127-6e12-5f29081acf2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" + testing.go:149: server-a4489edd-597d-3127-6e12-5f29081acf2f: 2024-01-08T10:38:01.797Z [INFO] server-a4489edd-597d-3127-6e12-5f29081acf2f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36} {Suffrage:Voter ID:server-a4489edd-597d-3127-6e12-5f29081acf2f Address:a4489edd-597d-3127-6e12-5f29081acf2f}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.797Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:01.797Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-21169459-7460-dcbb-5b04-fb17e5a32d36: 2024-01-08T10:38:01.798Z [INFO] server-21169459-7460-dcbb-5b04-fb17e5a32d36: entering follower state: follower="Node at 21169459-7460-dcbb-5b04-fb17e5a32d36 [Follower]" leader= + testing.go:149: server-21169459-7460-dcbb-5b04-fb17e5a32d36: 2024-01-08T10:38:01.798Z [INFO] server-21169459-7460-dcbb-5b04-fb17e5a32d36: entering follower state: follower="Node at 21169459-7460-dcbb-5b04-fb17e5a32d36 [Follower]" leader= +testLoggerAdapter verbose: server-a4489edd-597d-3127-6e12-5f29081acf2f: 2024-01-08T10:38:01.798Z [INFO] server-a4489edd-597d-3127-6e12-5f29081acf2f: entering follower state: follower="Node at a4489edd-597d-3127-6e12-5f29081acf2f [Follower]" leader= +testLoggerAdapter verbose: server-2690d475-d2db-301d-8d3e-5ec651a9bf47: 2024-01-08T10:38:01.798Z [INFO] server-2690d475-d2db-301d-8d3e-5ec651a9bf47: entering follower state: follower="Node at 2690d475-d2db-301d-8d3e-5ec651a9bf47 [Follower]" leader= + testing.go:149: server-a4489edd-597d-3127-6e12-5f29081acf2f: 2024-01-08T10:38:01.798Z [INFO] server-a4489edd-597d-3127-6e12-5f29081acf2f: entering follower state: follower="Node at a4489edd-597d-3127-6e12-5f29081acf2f [Follower]" leader= + testing.go:149: server-2690d475-d2db-301d-8d3e-5ec651a9bf47: 2024-01-08T10:38:01.798Z [INFO] server-2690d475-d2db-301d-8d3e-5ec651a9bf47: entering follower state: follower="Node at 2690d475-d2db-301d-8d3e-5ec651a9bf47 [Follower]" leader= +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.805Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering follower state: follower="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Follower]" leader= + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.805Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering follower state: follower="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Follower]" leader= +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.862Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.862Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.862Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering candidate state: node="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Candidate]" term=2 + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.862Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering candidate state: node="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Candidate]" term=2 +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: election won: tally=3 + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: election won: tally=3 +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering leader state: leader="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Leader]" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering leader state: leader="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Leader]" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-2690d475-d2db-301d-8d3e-5ec651a9bf47 + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-2690d475-d2db-301d-8d3e-5ec651a9bf47 +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-21169459-7460-dcbb-5b04-fb17e5a32d36 + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-21169459-7460-dcbb-5b04-fb17e5a32d36 +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-a4489edd-597d-3127-6e12-5f29081acf2f + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: added peer, starting replication: peer=server-a4489edd-597d-3127-6e12-5f29081acf2f +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-a4489edd-597d-3127-6e12-5f29081acf2f a4489edd-597d-3127-6e12-5f29081acf2f}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.863Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-a4489edd-597d-3127-6e12-5f29081acf2f a4489edd-597d-3127-6e12-5f29081acf2f}" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.864Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.864Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.864Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:01.864Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: pipelining replication: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:01.970Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.864363077 +0000 UTC m=+10.016157230 (1 nodes), 66.689184ms from start of poll, 70.279584ms from cluster start. Timeout at 2024-01-08 10:38:01.969948491 +0000 UTC m=+10.121742611, 105.585381ms after stability + testing.go:149: cluster: 2024-01-08T10:38:01.970Z [INFO] stable state for Leader reached at 2024-01-08 10:38:01.864363077 +0000 UTC m=+10.016157230 (1 nodes), 66.689184ms from start of poll, 70.279584ms from cluster start. Timeout at 2024-01-08 10:38:01.969948491 +0000 UTC m=+10.121742611, 105.585381ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.077Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:02.077Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.190Z [INFO] stable state for Follower reached at 2024-01-08 10:38:02.077973131 +0000 UTC m=+10.229766981 (3 nodes), 1.938µs from start of poll, 283.889335ms from cluster start. Timeout at 2024-01-08 10:38:02.189965125 +0000 UTC m=+10.341759306, 111.992325ms after stability + testing.go:149: cluster: 2024-01-08T10:38:02.190Z [INFO] stable state for Follower reached at 2024-01-08 10:38:02.077973131 +0000 UTC m=+10.229766981 (3 nodes), 1.938µs from start of poll, 283.889335ms from cluster start. Timeout at 2024-01-08 10:38:02.189965125 +0000 UTC m=+10.341759306, 111.992325ms after stability +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.190Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: updating configuration: command=RemoveServer server-id=server-a4489edd-597d-3127-6e12-5f29081acf2f server-addr= servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36}]" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.190Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: updating configuration: command=RemoveServer server-id=server-a4489edd-597d-3127-6e12-5f29081acf2f server-addr= servers="[{Suffrage:Voter ID:server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f Address:7f43a2d6-3266-2da2-1833-8f96c5e76f9f} {Suffrage:Voter ID:server-2690d475-d2db-301d-8d3e-5ec651a9bf47 Address:2690d475-d2db-301d-8d3e-5ec651a9bf47} {Suffrage:Voter ID:server-21169459-7460-dcbb-5b04-fb17e5a32d36 Address:21169459-7460-dcbb-5b04-fb17e5a32d36}]" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: removed peer, stopping replication: peer=server-a4489edd-597d-3127-6e12-5f29081acf2f last-index=3 + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: removed peer, stopping replication: peer=server-a4489edd-597d-3127-6e12-5f29081acf2f last-index=3 +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to pipeline appendEntries: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="append pipeline closed" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to pipeline appendEntries: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="append pipeline closed" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-a4489edd-597d-3127-6e12-5f29081acf2f a4489edd-597d-3127-6e12-5f29081acf2f}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-a4489edd-597d-3127-6e12-5f29081acf2f a4489edd-597d-3127-6e12-5f29081acf2f}" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to pipeline appendEntries: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="append pipeline closed" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to pipeline appendEntries: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="append pipeline closed" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: aborting pipeline replication: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.191Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.197Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.197Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.198Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=21169459-7460-dcbb-5b04-fb17e5a32d36 error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.198Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=21169459-7460-dcbb-5b04-fb17e5a32d36 error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.201Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.201Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.214Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.214Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.217Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.217Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.221Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.221Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.221Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=21169459-7460-dcbb-5b04-fb17e5a32d36 error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.221Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=21169459-7460-dcbb-5b04-fb17e5a32d36 error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.234Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.234Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to heartbeat to: peer=2690d475-d2db-301d-8d3e-5ec651a9bf47 error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.237Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.237Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-2690d475-d2db-301d-8d3e-5ec651a9bf47 2690d475-d2db-301d-8d3e-5ec651a9bf47}" error="failed to connect to peer: 2690d475-d2db-301d-8d3e-5ec651a9bf47" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.241Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.241Z [ERROR] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to appendEntries to: peer="{Voter server-21169459-7460-dcbb-5b04-fb17e5a32d36 21169459-7460-dcbb-5b04-fb17e5a32d36}" error="failed to connect to peer: 21169459-7460-dcbb-5b04-fb17e5a32d36" +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact: server-id=server-2690d475-d2db-301d-8d3e-5ec651a9bf47 time=52.196544ms + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact: server-id=server-2690d475-d2db-301d-8d3e-5ec651a9bf47 time=52.196544ms +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact: server-id=server-21169459-7460-dcbb-5b04-fb17e5a32d36 time=52.212357ms + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact: server-id=server-21169459-7460-dcbb-5b04-fb17e5a32d36 time=52.212357ms +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact quorum of nodes, stepping down + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [WARN] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering follower state: follower="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Follower]" leader= + testing.go:149: server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: 2024-01-08T10:38:02.242Z [INFO] server-7f43a2d6-3266-2da2-1833-8f96c5e76f9f: entering follower state: follower="Node at 7f43a2d6-3266-2da2-1833-8f96c5e76f9f [Follower]" leader= +--- PASS: TestRaft_RemoveFollower_SplitCluster (0.45s) === RUN TestRaft_AddKnownPeer -testLoggerAdapter verbose: server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: 2025-02-09T16:52:15.371Z [INFO] server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" - testing.go:149: server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: 2025-02-09T16:52:15.371Z [INFO] server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.371Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.371Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" -testLoggerAdapter verbose: server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: 2025-02-09T16:52:15.371Z [INFO] server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" - testing.go:149: server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: 2025-02-09T16:52:15.371Z [INFO] server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.371Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:15.371Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: 2025-02-09T16:52:15.371Z [INFO] server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: entering follower state: follower="Node at 3be3e906-9d8c-c3be-7a1e-e28ff224d947 [Follower]" leader= - testing.go:149: server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: 2025-02-09T16:52:15.371Z [INFO] server-3be3e906-9d8c-c3be-7a1e-e28ff224d947: entering follower state: follower="Node at 3be3e906-9d8c-c3be-7a1e-e28ff224d947 [Follower]" leader= -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.371Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering follower state: follower="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Follower]" leader= - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.371Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering follower state: follower="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Follower]" leader= -testLoggerAdapter verbose: server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: 2025-02-09T16:52:15.371Z [INFO] server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: entering follower state: follower="Node at c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 [Follower]" leader= - testing.go:149: server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: 2025-02-09T16:52:15.371Z [INFO] server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4: entering follower state: follower="Node at c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 [Follower]" leader= -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [WARN] server-49e70b91-7363-ff11-875c-c9d7155a2225: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [WARN] server-49e70b91-7363-ff11-875c-c9d7155a2225: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering candidate state: node="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Candidate]" term=2 - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering candidate state: node="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Candidate]" term=2 -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: election won: tally=2 - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: election won: tally=2 -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering leader state: leader="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Leader]" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.427Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: entering leader state: leader="Node at 49e70b91-7363-ff11-875c-c9d7155a2225 [Leader]" -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: added peer, starting replication: peer=server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: added peer, starting replication: peer=server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: added peer, starting replication: peer=server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: added peer, starting replication: peer=server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: pipelining replication: peer="{Voter server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: pipelining replication: peer="{Voter server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}" -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: pipelining replication: peer="{Voter server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 3be3e906-9d8c-c3be-7a1e-e28ff224d947}" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.428Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: pipelining replication: peer="{Voter server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 3be3e906-9d8c-c3be-7a1e-e28ff224d947}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.533Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.428278063 +0000 UTC m=+9.758510751 (1 nodes), 56.679621ms from start of poll, 57.043766ms from cluster start. Timeout at 2025-02-09 16:52:15.533340315 +0000 UTC m=+9.863573003, 105.062252ms after stability - testing.go:149: cluster: 2025-02-09T16:52:15.533Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.428278063 +0000 UTC m=+9.758510751 (1 nodes), 56.679621ms from start of poll, 57.043766ms from cluster start. Timeout at 2025-02-09 16:52:15.533340315 +0000 UTC m=+9.863573003, 105.062252ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.533Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:15.533Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.638Z [INFO] stable state for Follower reached at 2025-02-09 16:52:15.533447091 +0000 UTC m=+9.863679777 (2 nodes), 962ns from start of poll, 162.212792ms from cluster start. Timeout at 2025-02-09 16:52:15.638483846 +0000 UTC m=+9.968716534, 105.036757ms after stability - testing.go:149: cluster: 2025-02-09T16:52:15.638Z [INFO] stable state for Follower reached at 2025-02-09 16:52:15.533447091 +0000 UTC m=+9.863679777 (2 nodes), 962ns from start of poll, 162.212792ms from cluster start. Timeout at 2025-02-09 16:52:15.638483846 +0000 UTC m=+9.968716534, 105.036757ms after stability -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: updating configuration: command=AddStaging server-id=server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 server-addr=3be3e906-9d8c-c3be-7a1e-e28ff224d947 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: updating configuration: command=AddStaging server-id=server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 server-addr=3be3e906-9d8c-c3be-7a1e-e28ff224d947 servers="[{Suffrage:Voter ID:server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 Address:3be3e906-9d8c-c3be-7a1e-e28ff224d947} {Suffrage:Voter ID:server-49e70b91-7363-ff11-875c-c9d7155a2225 Address:49e70b91-7363-ff11-875c-c9d7155a2225} {Suffrage:Voter ID:server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 Address:c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}]" -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: aborting pipeline replication: peer="{Voter server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: aborting pipeline replication: peer="{Voter server-c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4 c47d2d9f-70ae-e21c-73fb-2f6ecd9860e4}" -testLoggerAdapter verbose: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: aborting pipeline replication: peer="{Voter server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 3be3e906-9d8c-c3be-7a1e-e28ff224d947}" - testing.go:149: server-49e70b91-7363-ff11-875c-c9d7155a2225: 2025-02-09T16:52:15.638Z [INFO] server-49e70b91-7363-ff11-875c-c9d7155a2225: aborting pipeline replication: peer="{Voter server-3be3e906-9d8c-c3be-7a1e-e28ff224d947 3be3e906-9d8c-c3be-7a1e-e28ff224d947}" ---- PASS: TestRaft_AddKnownPeer (0.27s) +testLoggerAdapter verbose: server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: 2024-01-08T10:38:02.248Z [INFO] server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" + testing.go:149: server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: 2024-01-08T10:38:02.248Z [INFO] server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" +testLoggerAdapter verbose: server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: 2024-01-08T10:38:02.249Z [INFO] server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" + testing.go:149: server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: 2024-01-08T10:38:02.249Z [INFO] server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" +testLoggerAdapter verbose: server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: 2024-01-08T10:38:02.249Z [INFO] server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: entering follower state: follower="Node at e32fa2a5-850a-2cc6-1983-26fdde8b73f9 [Follower]" leader= + testing.go:149: server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: 2024-01-08T10:38:02.249Z [INFO] server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9: entering follower state: follower="Node at e32fa2a5-850a-2cc6-1983-26fdde8b73f9 [Follower]" leader= +testLoggerAdapter verbose: server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: 2024-01-08T10:38:02.250Z [INFO] server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: entering follower state: follower="Node at 80e15918-92f5-b6dd-cfcc-6851d04b47e9 [Follower]" leader= + testing.go:149: server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: 2024-01-08T10:38:02.250Z [INFO] server-80e15918-92f5-b6dd-cfcc-6851d04b47e9: entering follower state: follower="Node at 80e15918-92f5-b6dd-cfcc-6851d04b47e9 [Follower]" leader= +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.258Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.258Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.259Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:02.259Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.259Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering follower state: follower="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Follower]" leader= + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.259Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering follower state: follower="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Follower]" leader= +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.310Z [WARN] server-4422c506-4a59-7a56-b46d-ac54f91a712f: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.310Z [WARN] server-4422c506-4a59-7a56-b46d-ac54f91a712f: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.310Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering candidate state: node="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Candidate]" term=2 + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.310Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering candidate state: node="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Candidate]" term=2 +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: election won: tally=2 + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: election won: tally=2 +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering leader state: leader="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Leader]" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: entering leader state: leader="Node at 4422c506-4a59-7a56-b46d-ac54f91a712f [Leader]" +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: added peer, starting replication: peer=server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.311Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: added peer, starting replication: peer=server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.312Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: added peer, starting replication: peer=server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.312Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: added peer, starting replication: peer=server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.312Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: pipelining replication: peer="{Voter server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 80e15918-92f5-b6dd-cfcc-6851d04b47e9}" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.312Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: pipelining replication: peer="{Voter server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 80e15918-92f5-b6dd-cfcc-6851d04b47e9}" +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.313Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: pipelining replication: peer="{Voter server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 e32fa2a5-850a-2cc6-1983-26fdde8b73f9}" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.313Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: pipelining replication: peer="{Voter server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 e32fa2a5-850a-2cc6-1983-26fdde8b73f9}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.422Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.313430057 +0000 UTC m=+10.465224214 (1 nodes), 54.188788ms from start of poll, 65.550585ms from cluster start. Timeout at 2024-01-08 10:38:02.421926804 +0000 UTC m=+10.573720950, 108.496736ms after stability + testing.go:149: cluster: 2024-01-08T10:38:02.422Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.313430057 +0000 UTC m=+10.465224214 (1 nodes), 54.188788ms from start of poll, 65.550585ms from cluster start. Timeout at 2024-01-08 10:38:02.421926804 +0000 UTC m=+10.573720950, 108.496736ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.422Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:02.422Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.530Z [INFO] stable state for Follower reached at 2024-01-08 10:38:02.42294832 +0000 UTC m=+10.574742444 (2 nodes), 4.904µs from start of poll, 175.068815ms from cluster start. Timeout at 2024-01-08 10:38:02.530213593 +0000 UTC m=+10.682007720, 107.265276ms after stability + testing.go:149: cluster: 2024-01-08T10:38:02.530Z [INFO] stable state for Follower reached at 2024-01-08 10:38:02.42294832 +0000 UTC m=+10.574742444 (2 nodes), 4.904µs from start of poll, 175.068815ms from cluster start. Timeout at 2024-01-08 10:38:02.530213593 +0000 UTC m=+10.682007720, 107.265276ms after stability +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.530Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: updating configuration: command=AddStaging server-id=server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 server-addr=e32fa2a5-850a-2cc6-1983-26fdde8b73f9 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.530Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: updating configuration: command=AddStaging server-id=server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 server-addr=e32fa2a5-850a-2cc6-1983-26fdde8b73f9 servers="[{Suffrage:Voter ID:server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 Address:e32fa2a5-850a-2cc6-1983-26fdde8b73f9} {Suffrage:Voter ID:server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 Address:80e15918-92f5-b6dd-cfcc-6851d04b47e9} {Suffrage:Voter ID:server-4422c506-4a59-7a56-b46d-ac54f91a712f Address:4422c506-4a59-7a56-b46d-ac54f91a712f}]" +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.532Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: aborting pipeline replication: peer="{Voter server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 80e15918-92f5-b6dd-cfcc-6851d04b47e9}" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.532Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: aborting pipeline replication: peer="{Voter server-80e15918-92f5-b6dd-cfcc-6851d04b47e9 80e15918-92f5-b6dd-cfcc-6851d04b47e9}" +testLoggerAdapter verbose: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.532Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: aborting pipeline replication: peer="{Voter server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 e32fa2a5-850a-2cc6-1983-26fdde8b73f9}" + testing.go:149: server-4422c506-4a59-7a56-b46d-ac54f91a712f: 2024-01-08T10:38:02.532Z [INFO] server-4422c506-4a59-7a56-b46d-ac54f91a712f: aborting pipeline replication: peer="{Voter server-e32fa2a5-850a-2cc6-1983-26fdde8b73f9 e32fa2a5-850a-2cc6-1983-26fdde8b73f9}" +--- PASS: TestRaft_AddKnownPeer (0.29s) === RUN TestRaft_RemoveUnknownPeer -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.639Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.639Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" -testLoggerAdapter verbose: server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: 2025-02-09T16:52:15.639Z [INFO] server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" - testing.go:149: server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: 2025-02-09T16:52:15.639Z [INFO] server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" -testLoggerAdapter verbose: server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: 2025-02-09T16:52:15.639Z [INFO] server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" - testing.go:149: server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: 2025-02-09T16:52:15.639Z [INFO] server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.639Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:15.639Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.639Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering follower state: follower="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Follower]" leader= - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.639Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering follower state: follower="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Follower]" leader= -testLoggerAdapter verbose: server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: 2025-02-09T16:52:15.639Z [INFO] server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: entering follower state: follower="Node at fe153dac-b9fd-a9f3-ae04-d729020bf2c3 [Follower]" leader= - testing.go:149: server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: 2025-02-09T16:52:15.639Z [INFO] server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3: entering follower state: follower="Node at fe153dac-b9fd-a9f3-ae04-d729020bf2c3 [Follower]" leader= -testLoggerAdapter verbose: server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: 2025-02-09T16:52:15.639Z [INFO] server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: entering follower state: follower="Node at 5f9c0d98-a09e-0d59-cb08-c743b9bf109a [Follower]" leader= - testing.go:149: server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: 2025-02-09T16:52:15.639Z [INFO] server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a: entering follower state: follower="Node at 5f9c0d98-a09e-0d59-cb08-c743b9bf109a [Follower]" leader= -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [WARN] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [WARN] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering candidate state: node="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Candidate]" term=2 - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering candidate state: node="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Candidate]" term=2 -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: election won: tally=2 - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: election won: tally=2 -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering leader state: leader="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Leader]" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: entering leader state: leader="Node at 7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 [Leader]" -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: added peer, starting replication: peer=server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: added peer, starting replication: peer=server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: added peer, starting replication: peer=server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.696Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: added peer, starting replication: peer=server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.697Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: pipelining replication: peer="{Voter server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a 5f9c0d98-a09e-0d59-cb08-c743b9bf109a}" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.697Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: pipelining replication: peer="{Voter server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a 5f9c0d98-a09e-0d59-cb08-c743b9bf109a}" -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.697Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: pipelining replication: peer="{Voter server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 fe153dac-b9fd-a9f3-ae04-d729020bf2c3}" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.697Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: pipelining replication: peer="{Voter server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 fe153dac-b9fd-a9f3-ae04-d729020bf2c3}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.802Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.69723948 +0000 UTC m=+10.027472168 (1 nodes), 57.542188ms from start of poll, 57.850002ms from cluster start. Timeout at 2025-02-09 16:52:15.802310342 +0000 UTC m=+10.132543029, 105.070861ms after stability - testing.go:149: cluster: 2025-02-09T16:52:15.802Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.69723948 +0000 UTC m=+10.027472168 (1 nodes), 57.542188ms from start of poll, 57.850002ms from cluster start. Timeout at 2025-02-09 16:52:15.802310342 +0000 UTC m=+10.132543029, 105.070861ms after stability -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: updating configuration: command=RemoveServer server-id=d404cf5a-616f-bdc4-300c-ee60b5123120 server-addr= servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: updating configuration: command=RemoveServer server-id=d404cf5a-616f-bdc4-300c-ee60b5123120 server-addr= servers="[{Suffrage:Voter ID:server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6 Address:7b56c9ae-b6dd-44ac-0348-b4ba927e47a6} {Suffrage:Voter ID:server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 Address:fe153dac-b9fd-a9f3-ae04-d729020bf2c3} {Suffrage:Voter ID:server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a Address:5f9c0d98-a09e-0d59-cb08-c743b9bf109a}]" -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: aborting pipeline replication: peer="{Voter server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a 5f9c0d98-a09e-0d59-cb08-c743b9bf109a}" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: aborting pipeline replication: peer="{Voter server-5f9c0d98-a09e-0d59-cb08-c743b9bf109a 5f9c0d98-a09e-0d59-cb08-c743b9bf109a}" -testLoggerAdapter verbose: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: aborting pipeline replication: peer="{Voter server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 fe153dac-b9fd-a9f3-ae04-d729020bf2c3}" - testing.go:149: server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: 2025-02-09T16:52:15.802Z [INFO] server-7b56c9ae-b6dd-44ac-0348-b4ba927e47a6: aborting pipeline replication: peer="{Voter server-fe153dac-b9fd-a9f3-ae04-d729020bf2c3 fe153dac-b9fd-a9f3-ae04-d729020bf2c3}" ---- PASS: TestRaft_RemoveUnknownPeer (0.16s) +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.537Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.537Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.538Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering follower state: follower="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Follower]" leader= + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.538Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering follower state: follower="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Follower]" leader= +testLoggerAdapter verbose: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.538Z [INFO] server-9bb34778-c256-a797-dbc1-ced20d69e690: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" + testing.go:149: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.538Z [INFO] server-9bb34778-c256-a797-dbc1-ced20d69e690: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" +testLoggerAdapter verbose: server-40ce7698-4390-7e03-4047-621fbe0b0290: 2024-01-08T10:38:02.539Z [INFO] server-40ce7698-4390-7e03-4047-621fbe0b0290: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" + testing.go:149: server-40ce7698-4390-7e03-4047-621fbe0b0290: 2024-01-08T10:38:02.539Z [INFO] server-40ce7698-4390-7e03-4047-621fbe0b0290: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.539Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:02.539Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.539Z [INFO] server-9bb34778-c256-a797-dbc1-ced20d69e690: entering follower state: follower="Node at 9bb34778-c256-a797-dbc1-ced20d69e690 [Follower]" leader= + testing.go:149: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.539Z [INFO] server-9bb34778-c256-a797-dbc1-ced20d69e690: entering follower state: follower="Node at 9bb34778-c256-a797-dbc1-ced20d69e690 [Follower]" leader= +testLoggerAdapter verbose: server-40ce7698-4390-7e03-4047-621fbe0b0290: 2024-01-08T10:38:02.540Z [INFO] server-40ce7698-4390-7e03-4047-621fbe0b0290: entering follower state: follower="Node at 40ce7698-4390-7e03-4047-621fbe0b0290 [Follower]" leader= + testing.go:149: server-40ce7698-4390-7e03-4047-621fbe0b0290: 2024-01-08T10:38:02.540Z [INFO] server-40ce7698-4390-7e03-4047-621fbe0b0290: entering follower state: follower="Node at 40ce7698-4390-7e03-4047-621fbe0b0290 [Follower]" leader= +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.591Z [WARN] server-5788c91d-83af-7a39-eac1-3822769cb799: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.591Z [WARN] server-5788c91d-83af-7a39-eac1-3822769cb799: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering candidate state: node="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Candidate]" term=2 + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering candidate state: node="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Candidate]" term=2 +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: election won: tally=2 + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: election won: tally=2 +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering leader state: leader="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Leader]" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: entering leader state: leader="Node at 5788c91d-83af-7a39-eac1-3822769cb799 [Leader]" +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: added peer, starting replication: peer=server-9bb34778-c256-a797-dbc1-ced20d69e690 + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.592Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: added peer, starting replication: peer=server-9bb34778-c256-a797-dbc1-ced20d69e690 +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.593Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: added peer, starting replication: peer=server-40ce7698-4390-7e03-4047-621fbe0b0290 + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.593Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: added peer, starting replication: peer=server-40ce7698-4390-7e03-4047-621fbe0b0290 +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.593Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: pipelining replication: peer="{Voter server-40ce7698-4390-7e03-4047-621fbe0b0290 40ce7698-4390-7e03-4047-621fbe0b0290}" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.593Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: pipelining replication: peer="{Voter server-40ce7698-4390-7e03-4047-621fbe0b0290 40ce7698-4390-7e03-4047-621fbe0b0290}" +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.594Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: pipelining replication: peer="{Voter server-9bb34778-c256-a797-dbc1-ced20d69e690 9bb34778-c256-a797-dbc1-ced20d69e690}" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.594Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: pipelining replication: peer="{Voter server-9bb34778-c256-a797-dbc1-ced20d69e690 9bb34778-c256-a797-dbc1-ced20d69e690}" +testLoggerAdapter verbose: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.594Z [WARN] server-9bb34778-c256-a797-dbc1-ced20d69e690: rejecting vote request since our last term is greater: candidate=5788c91d-83af-7a39-eac1-3822769cb799 last-term=2 last-candidate-term=1 + testing.go:149: server-9bb34778-c256-a797-dbc1-ced20d69e690: 2024-01-08T10:38:02.594Z [WARN] server-9bb34778-c256-a797-dbc1-ced20d69e690: rejecting vote request since our last term is greater: candidate=5788c91d-83af-7a39-eac1-3822769cb799 last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.702Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.594352253 +0000 UTC m=+10.746146104 (1 nodes), 54.647871ms from start of poll, 57.546462ms from cluster start. Timeout at 2024-01-08 10:38:02.702537152 +0000 UTC m=+10.854331270, 108.185166ms after stability + testing.go:149: cluster: 2024-01-08T10:38:02.702Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.594352253 +0000 UTC m=+10.746146104 (1 nodes), 54.647871ms from start of poll, 57.546462ms from cluster start. Timeout at 2024-01-08 10:38:02.702537152 +0000 UTC m=+10.854331270, 108.185166ms after stability +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.703Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: updating configuration: command=RemoveServer server-id=87ec7aef-9de0-91c3-ed6d-282494dd1528 server-addr= servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.703Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: updating configuration: command=RemoveServer server-id=87ec7aef-9de0-91c3-ed6d-282494dd1528 server-addr= servers="[{Suffrage:Voter ID:server-5788c91d-83af-7a39-eac1-3822769cb799 Address:5788c91d-83af-7a39-eac1-3822769cb799} {Suffrage:Voter ID:server-9bb34778-c256-a797-dbc1-ced20d69e690 Address:9bb34778-c256-a797-dbc1-ced20d69e690} {Suffrage:Voter ID:server-40ce7698-4390-7e03-4047-621fbe0b0290 Address:40ce7698-4390-7e03-4047-621fbe0b0290}]" +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.721Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: aborting pipeline replication: peer="{Voter server-40ce7698-4390-7e03-4047-621fbe0b0290 40ce7698-4390-7e03-4047-621fbe0b0290}" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.721Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: aborting pipeline replication: peer="{Voter server-40ce7698-4390-7e03-4047-621fbe0b0290 40ce7698-4390-7e03-4047-621fbe0b0290}" +testLoggerAdapter verbose: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.721Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: aborting pipeline replication: peer="{Voter server-9bb34778-c256-a797-dbc1-ced20d69e690 9bb34778-c256-a797-dbc1-ced20d69e690}" + testing.go:149: server-5788c91d-83af-7a39-eac1-3822769cb799: 2024-01-08T10:38:02.721Z [INFO] server-5788c91d-83af-7a39-eac1-3822769cb799: aborting pipeline replication: peer="{Voter server-9bb34778-c256-a797-dbc1-ced20d69e690 9bb34778-c256-a797-dbc1-ced20d69e690}" +--- PASS: TestRaft_RemoveUnknownPeer (0.19s) === RUN TestRaft_SnapshotRestore -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.803Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fa9bf50-64e2-9b92-32a3-b9224f66779f Address:8fa9bf50-64e2-9b92-32a3-b9224f66779f}]" - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.803Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fa9bf50-64e2-9b92-32a3-b9224f66779f Address:8fa9bf50-64e2-9b92-32a3-b9224f66779f}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:15.803Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:15.803Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.803Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering follower state: follower="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Follower]" leader= - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.803Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering follower state: follower="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Follower]" leader= -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [WARN] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [WARN] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering candidate state: node="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Candidate]" term=2 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering candidate state: node="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Candidate]" term=2 -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: election won: tally=1 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.894Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: election won: tally=1 -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.895Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering leader state: leader="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Leader]" - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:15.895Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: entering leader state: leader="Node at 8fa9bf50-64e2-9b92-32a3-b9224f66779f [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.000Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.895053455 +0000 UTC m=+10.225286142 (1 nodes), 91.826717ms from start of poll, 91.9664ms from cluster start. Timeout at 2025-02-09 16:52:16.00068491 +0000 UTC m=+10.330917615, 105.631473ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.000Z [INFO] stable state for Leader reached at 2025-02-09 16:52:15.895053455 +0000 UTC m=+10.225286142 (1 nodes), 91.826717ms from start of poll, 91.9664ms from cluster start. Timeout at 2025-02-09 16:52:16.00068491 +0000 UTC m=+10.330917615, 105.631473ms after stability -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.001Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: starting snapshot up to: index=102 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.001Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T16:52:16.001Z [INFO] creating new snapshot: path=/tmp/raft326523187/snapshots/2-102-1739119936001.tmp -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: compacting logs: from=1 to=92 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: snapshot complete up to: index=102 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: restored from snapshot: id=2-102-1739119936001 - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: restored from snapshot: id=2-102-1739119936001 -testLoggerAdapter verbose: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fa9bf50-64e2-9b92-32a3-b9224f66779f Address:8fa9bf50-64e2-9b92-32a3-b9224f66779f}]" - testing.go:149: server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: 2025-02-09T16:52:16.002Z [INFO] server-8fa9bf50-64e2-9b92-32a3-b9224f66779f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8fa9bf50-64e2-9b92-32a3-b9224f66779f Address:8fa9bf50-64e2-9b92-32a3-b9224f66779f}]" ---- PASS: TestRaft_SnapshotRestore (0.20s) +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.734Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba3a5a6-8dde-359e-af29-debeaec619c0 Address:6ba3a5a6-8dde-359e-af29-debeaec619c0}]" + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.734Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba3a5a6-8dde-359e-af29-debeaec619c0 Address:6ba3a5a6-8dde-359e-af29-debeaec619c0}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.734Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:02.734Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.734Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering follower state: follower="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Follower]" leader= + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.734Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering follower state: follower="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Follower]" leader= +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.810Z [WARN] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.810Z [WARN] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.810Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering candidate state: node="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Candidate]" term=2 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.810Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering candidate state: node="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Candidate]" term=2 +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.811Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: election won: tally=1 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.811Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: election won: tally=1 +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.811Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering leader state: leader="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Leader]" + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.811Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: entering leader state: leader="Node at 6ba3a5a6-8dde-359e-af29-debeaec619c0 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.918Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.81167285 +0000 UTC m=+10.963466985 (1 nodes), 77.103265ms from start of poll, 86.457391ms from cluster start. Timeout at 2024-01-08 10:38:02.917742145 +0000 UTC m=+11.069536291, 106.069306ms after stability + testing.go:149: cluster: 2024-01-08T10:38:02.918Z [INFO] stable state for Leader reached at 2024-01-08 10:38:02.81167285 +0000 UTC m=+10.963466985 (1 nodes), 77.103265ms from start of poll, 86.457391ms from cluster start. Timeout at 2024-01-08 10:38:02.917742145 +0000 UTC m=+11.069536291, 106.069306ms after stability +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.928Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: starting snapshot up to: index=102 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.928Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T10:38:02.929Z [INFO] creating new snapshot: path=/tmp/raft1501555841/snapshots/2-102-1704710282929.tmp +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.935Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: compacting logs: from=1 to=92 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.935Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.935Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: snapshot complete up to: index=102 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.935Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.937Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: restored from snapshot: id=2-102-1704710282929 + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.937Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: restored from snapshot: id=2-102-1704710282929 +testLoggerAdapter verbose: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.937Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba3a5a6-8dde-359e-af29-debeaec619c0 Address:6ba3a5a6-8dde-359e-af29-debeaec619c0}]" + testing.go:149: server-6ba3a5a6-8dde-359e-af29-debeaec619c0: 2024-01-08T10:38:02.937Z [INFO] server-6ba3a5a6-8dde-359e-af29-debeaec619c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ba3a5a6-8dde-359e-af29-debeaec619c0 Address:6ba3a5a6-8dde-359e-af29-debeaec619c0}]" +--- PASS: TestRaft_SnapshotRestore (0.23s) === RUN TestRaft_NoRestoreOnStart -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.002Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67234341-ebc4-aabf-4cd3-de730dca4aaa Address:67234341-ebc4-aabf-4cd3-de730dca4aaa}]" - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.002Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67234341-ebc4-aabf-4cd3-de730dca4aaa Address:67234341-ebc4-aabf-4cd3-de730dca4aaa}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.002Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.002Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.002Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering follower state: follower="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Follower]" leader= - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.002Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering follower state: follower="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Follower]" leader= -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [WARN] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [WARN] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering candidate state: node="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Candidate]" term=2 - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering candidate state: node="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Candidate]" term=2 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: election won: tally=1 - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: election won: tally=1 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering leader state: leader="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Leader]" - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.060Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering leader state: leader="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.166Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.060643101 +0000 UTC m=+10.390875789 (1 nodes), 57.922133ms from start of poll, 58.036825ms from cluster start. Timeout at 2025-02-09 16:52:16.166356145 +0000 UTC m=+10.496588833, 105.713044ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.166Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.060643101 +0000 UTC m=+10.390875789 (1 nodes), 57.922133ms from start of poll, 58.036825ms from cluster start. Timeout at 2025-02-09 16:52:16.166356145 +0000 UTC m=+10.496588833, 105.713044ms after stability -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: starting snapshot up to: index=102 - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T16:52:16.167Z [INFO] creating new snapshot: path=/tmp/raft2285752075/snapshots/2-102-1739119936167.tmp -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: compacting logs: from=1 to=92 - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: snapshot complete up to: index=102 - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67234341-ebc4-aabf-4cd3-de730dca4aaa Address:67234341-ebc4-aabf-4cd3-de730dca4aaa}]" - testing.go:149: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.167Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-67234341-ebc4-aabf-4cd3-de730dca4aaa Address:67234341-ebc4-aabf-4cd3-de730dca4aaa}]" ---- PASS: TestRaft_NoRestoreOnStart (0.17s) +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:02.953Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3510b723-9adc-80e1-6b6a-c301169d94db Address:3510b723-9adc-80e1-6b6a-c301169d94db}]" + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:02.953Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3510b723-9adc-80e1-6b6a-c301169d94db Address:3510b723-9adc-80e1-6b6a-c301169d94db}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:02.953Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:02.953Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:02.953Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering follower state: follower="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Follower]" leader= + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:02.953Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering follower state: follower="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Follower]" leader= +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [WARN] server-3510b723-9adc-80e1-6b6a-c301169d94db: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [WARN] server-3510b723-9adc-80e1-6b6a-c301169d94db: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering candidate state: node="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Candidate]" term=2 + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering candidate state: node="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Candidate]" term=2 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: election won: tally=1 + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: election won: tally=1 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering leader state: leader="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Leader]" + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.030Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering leader state: leader="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.138Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.031065279 +0000 UTC m=+11.182859417 (1 nodes), 77.562918ms from start of poll, 78.327548ms from cluster start. Timeout at 2024-01-08 10:38:03.137944111 +0000 UTC m=+11.289738279, 106.878862ms after stability + testing.go:149: cluster: 2024-01-08T10:38:03.138Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.031065279 +0000 UTC m=+11.182859417 (1 nodes), 77.562918ms from start of poll, 78.327548ms from cluster start. Timeout at 2024-01-08 10:38:03.137944111 +0000 UTC m=+11.289738279, 106.878862ms after stability +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.144Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: starting snapshot up to: index=102 + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.144Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T10:38:03.144Z [INFO] creating new snapshot: path=/tmp/raft3618615501/snapshots/2-102-1704710283144.tmp +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.148Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: compacting logs: from=1 to=92 + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.148Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.148Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: snapshot complete up to: index=102 + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.148Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.149Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3510b723-9adc-80e1-6b6a-c301169d94db Address:3510b723-9adc-80e1-6b6a-c301169d94db}]" + testing.go:149: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.149Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3510b723-9adc-80e1-6b6a-c301169d94db Address:3510b723-9adc-80e1-6b6a-c301169d94db}]" +--- PASS: TestRaft_NoRestoreOnStart (0.20s) === RUN TestRaft_SnapshotRestore_PeerChange raft_test.go:1026: Disabled unreliable test --- SKIP: TestRaft_SnapshotRestore_PeerChange (0.00s) @@ -2105,1749 +2143,1783 @@ raft_test.go:1132: Disabled unreliable test --- SKIP: TestRaft_AutoSnapshot (0.00s) === RUN TestRaft_UserSnapshot -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.168Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a76491d6-1dee-f059-61c7-0bc958d5ef63 Address:a76491d6-1dee-f059-61c7-0bc958d5ef63}]" - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.168Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a76491d6-1dee-f059-61c7-0bc958d5ef63 Address:a76491d6-1dee-f059-61c7-0bc958d5ef63}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.168Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.168Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.168Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering follower state: follower="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Follower]" leader= -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.168Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering follower state: follower="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Follower]" leader= - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.168Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering follower state: follower="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Follower]" leader= -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.224Z [WARN] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.224Z [WARN] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering candidate state: node="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Candidate]" term=2 - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering candidate state: node="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Candidate]" term=2 -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: election won: tally=1 - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: election won: tally=1 -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering leader state: leader="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Leader]" - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.225Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: entering leader state: leader="Node at a76491d6-1dee-f059-61c7-0bc958d5ef63 [Leader]" -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.258Z [WARN] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.258Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering candidate state: node="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Candidate]" term=3 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.258Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: election won: tally=1 -testLoggerAdapter verbose: server-67234341-ebc4-aabf-4cd3-de730dca4aaa: 2025-02-09T16:52:16.258Z [INFO] server-67234341-ebc4-aabf-4cd3-de730dca4aaa: entering leader state: leader="Node at 67234341-ebc4-aabf-4cd3-de730dca4aaa [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.330Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.225162553 +0000 UTC m=+10.555395241 (1 nodes), 57.045959ms from start of poll, 57.201733ms from cluster start. Timeout at 2025-02-09 16:52:16.330182933 +0000 UTC m=+10.660415621, 105.02038ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.330Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.225162553 +0000 UTC m=+10.555395241 (1 nodes), 57.045959ms from start of poll, 57.201733ms from cluster start. Timeout at 2025-02-09 16:52:16.330182933 +0000 UTC m=+10.660415621, 105.02038ms after stability -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [ERROR] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [ERROR] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: starting snapshot up to: index=12 - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:16.330Z [INFO] creating new snapshot: path=/tmp/raft2356388670/snapshots/2-12-1739119936330.tmp -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: compacting logs: from=1 to=2 - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: compacting logs: from=1 to=2 -testLoggerAdapter verbose: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: snapshot complete up to: index=12 - testing.go:149: server-a76491d6-1dee-f059-61c7-0bc958d5ef63: 2025-02-09T16:52:16.330Z [INFO] server-a76491d6-1dee-f059-61c7-0bc958d5ef63: snapshot complete up to: index=12 ---- PASS: TestRaft_UserSnapshot (0.16s) +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.152Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-176652f0-1cee-e0cf-daf0-9375065c0d28 Address:176652f0-1cee-e0cf-daf0-9375065c0d28}]" + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.152Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-176652f0-1cee-e0cf-daf0-9375065c0d28 Address:176652f0-1cee-e0cf-daf0-9375065c0d28}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.152Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:03.152Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.152Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering follower state: follower="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Follower]" leader= +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.153Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering follower state: follower="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Follower]" leader= + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.153Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering follower state: follower="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Follower]" leader= +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [WARN] server-176652f0-1cee-e0cf-daf0-9375065c0d28: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [WARN] server-176652f0-1cee-e0cf-daf0-9375065c0d28: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering candidate state: node="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Candidate]" term=2 + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering candidate state: node="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Candidate]" term=2 +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: election won: tally=1 + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: election won: tally=1 +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering leader state: leader="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Leader]" + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.234Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: entering leader state: leader="Node at 176652f0-1cee-e0cf-daf0-9375065c0d28 [Leader]" +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.253Z [WARN] server-3510b723-9adc-80e1-6b6a-c301169d94db: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.254Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering candidate state: node="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Candidate]" term=3 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.254Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: election won: tally=1 +testLoggerAdapter verbose: server-3510b723-9adc-80e1-6b6a-c301169d94db: 2024-01-08T10:38:03.254Z [INFO] server-3510b723-9adc-80e1-6b6a-c301169d94db: entering leader state: leader="Node at 3510b723-9adc-80e1-6b6a-c301169d94db [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.342Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.234934266 +0000 UTC m=+11.386728432 (1 nodes), 82.111989ms from start of poll, 83.177219ms from cluster start. Timeout at 2024-01-08 10:38:03.341900654 +0000 UTC m=+11.493694786, 106.966354ms after stability + testing.go:149: cluster: 2024-01-08T10:38:03.342Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.234934266 +0000 UTC m=+11.386728432 (1 nodes), 82.111989ms from start of poll, 83.177219ms from cluster start. Timeout at 2024-01-08 10:38:03.341900654 +0000 UTC m=+11.493694786, 106.966354ms after stability +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.342Z [ERROR] server-176652f0-1cee-e0cf-daf0-9375065c0d28: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.342Z [ERROR] server-176652f0-1cee-e0cf-daf0-9375065c0d28: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.343Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: starting snapshot up to: index=12 + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.343Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:03.344Z [INFO] creating new snapshot: path=/tmp/raft2020843945/snapshots/2-12-1704710283344.tmp +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.346Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: compacting logs: from=1 to=2 + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.346Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: compacting logs: from=1 to=2 +testLoggerAdapter verbose: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.347Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: snapshot complete up to: index=12 + testing.go:149: server-176652f0-1cee-e0cf-daf0-9375065c0d28: 2024-01-08T10:38:03.347Z [INFO] server-176652f0-1cee-e0cf-daf0-9375065c0d28: snapshot complete up to: index=12 +--- PASS: TestRaft_UserSnapshot (0.20s) === RUN TestRaft_UserRestore -testLoggerAdapter verbose: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.331Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" - testing.go:149: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.331Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" -testLoggerAdapter verbose: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.331Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" - testing.go:149: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.331Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.331Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.331Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 Address:823f772a-eb53-9ad3-91d8-2b96aab1ef25} {Suffrage:Voter ID:server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 Address:5b423e04-87c9-fe4d-07c4-6567a5ebee76} {Suffrage:Voter ID:server-1bec18b5-108d-b581-be1b-9ce0dfad75b3 Address:1bec18b5-108d-b581-be1b-9ce0dfad75b3}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.331Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.331Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.331Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: entering follower state: follower="Node at 823f772a-eb53-9ad3-91d8-2b96aab1ef25 [Follower]" leader= - testing.go:149: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.331Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: entering follower state: follower="Node at 823f772a-eb53-9ad3-91d8-2b96aab1ef25 [Follower]" leader= -testLoggerAdapter verbose: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.331Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: entering follower state: follower="Node at 5b423e04-87c9-fe4d-07c4-6567a5ebee76 [Follower]" leader= - testing.go:149: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.331Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: entering follower state: follower="Node at 5b423e04-87c9-fe4d-07c4-6567a5ebee76 [Follower]" leader= -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.331Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering follower state: follower="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Follower]" leader= - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.331Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering follower state: follower="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Follower]" leader= -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.404Z [WARN] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.404Z [WARN] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering candidate state: node="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Candidate]" term=2 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering candidate state: node="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Candidate]" term=2 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: election won: tally=2 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: election won: tally=2 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering leader state: leader="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Leader]" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: entering leader state: leader="Node at 1bec18b5-108d-b581-be1b-9ce0dfad75b3 [Leader]" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: added peer, starting replication: peer=server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: added peer, starting replication: peer=server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: added peer, starting replication: peer=server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: added peer, starting replication: peer=server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.405Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.511Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.405433353 +0000 UTC m=+10.735666041 (1 nodes), 73.75506ms from start of poll, 74.107142ms from cluster start. Timeout at 2025-02-09 16:52:16.510978316 +0000 UTC m=+10.841211003, 105.544962ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.511Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.405433353 +0000 UTC m=+10.735666041 (1 nodes), 73.75506ms from start of poll, 74.107142ms from cluster start. Timeout at 2025-02-09 16:52:16.510978316 +0000 UTC m=+10.841211003, 105.544962ms after stability -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.511Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: starting snapshot up to: index=12 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.511Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:16.511Z [INFO] creating new snapshot: path=/tmp/raft3741098675/snapshots/2-12-1739119936511.tmp -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.512Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: snapshot complete up to: index=12 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.512Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:16.514Z [INFO] creating new snapshot: path=/tmp/raft3741098675/snapshots/2-23-1739119936514.tmp -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.514Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: copied to local snapshot: bytes=71 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.514Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: restored user snapshot: index=1 - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: restored user snapshot: index=1 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.515Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.521Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.521Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:16.522Z [INFO] creating new snapshot: path=/tmp/raft1697805986/snapshots/2-23-1739119936522.tmp -testLoggerAdapter verbose: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: copied to local snapshot: bytes=71 - testing.go:149: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: Installed remote snapshot - testing.go:149: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [INFO] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: Installed remote snapshot -testLoggerAdapter verbose: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [WARN] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: skipping application of old log: index=22 - testing.go:149: server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: 2025-02-09T16:52:16.522Z [WARN] server-823f772a-eb53-9ad3-91d8-2b96aab1ef25: skipping application of old log: index=22 -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.522Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.522Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.524Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.524Z [ERROR] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:16.524Z [INFO] creating new snapshot: path=/tmp/raft2337416777/snapshots/2-23-1739119936524.tmp -testLoggerAdapter verbose: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.525Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: copied to local snapshot: bytes=71 - testing.go:149: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.525Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.525Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: Installed remote snapshot - testing.go:149: server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: 2025-02-09T16:52:16.525Z [INFO] server-5b423e04-87c9-fe4d-07c4-6567a5ebee76: Installed remote snapshot -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.525Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.525Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: pipelining replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.540Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.540Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-5b423e04-87c9-fe4d-07c4-6567a5ebee76 5b423e04-87c9-fe4d-07c4-6567a5ebee76}" -testLoggerAdapter verbose: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.540Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" - testing.go:149: server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: 2025-02-09T16:52:16.540Z [INFO] server-1bec18b5-108d-b581-be1b-9ce0dfad75b3: aborting pipeline replication: peer="{Voter server-823f772a-eb53-9ad3-91d8-2b96aab1ef25 823f772a-eb53-9ad3-91d8-2b96aab1ef25}" -testLoggerAdapter verbose: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.540Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" - testing.go:149: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.540Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.540Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.540Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" -testLoggerAdapter verbose: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.541Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" - testing.go:149: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.541Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-86ebc7ac-66bd-4408-8264-e0cd2136da97 Address:86ebc7ac-66bd-4408-8264-e0cd2136da97} {Suffrage:Voter ID:server-aff24f1d-7182-8979-02c3-856a74358352 Address:aff24f1d-7182-8979-02c3-856a74358352} {Suffrage:Voter ID:server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef Address:2f7f262d-e356-1e68-d5e2-9a64e844f2ef}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.541Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.541Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.541Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: entering follower state: follower="Node at 86ebc7ac-66bd-4408-8264-e0cd2136da97 [Follower]" leader= - testing.go:149: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.541Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: entering follower state: follower="Node at 86ebc7ac-66bd-4408-8264-e0cd2136da97 [Follower]" leader= -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.541Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering follower state: follower="Node at aff24f1d-7182-8979-02c3-856a74358352 [Follower]" leader= - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.541Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering follower state: follower="Node at aff24f1d-7182-8979-02c3-856a74358352 [Follower]" leader= -testLoggerAdapter verbose: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.541Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: entering follower state: follower="Node at 2f7f262d-e356-1e68-d5e2-9a64e844f2ef [Follower]" leader= - testing.go:149: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.541Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: entering follower state: follower="Node at 2f7f262d-e356-1e68-d5e2-9a64e844f2ef [Follower]" leader= -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [WARN] server-aff24f1d-7182-8979-02c3-856a74358352: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [WARN] server-aff24f1d-7182-8979-02c3-856a74358352: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering candidate state: node="Node at aff24f1d-7182-8979-02c3-856a74358352 [Candidate]" term=2 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering candidate state: node="Node at aff24f1d-7182-8979-02c3-856a74358352 [Candidate]" term=2 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: election won: tally=2 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: election won: tally=2 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering leader state: leader="Node at aff24f1d-7182-8979-02c3-856a74358352 [Leader]" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: entering leader state: leader="Node at aff24f1d-7182-8979-02c3-856a74358352 [Leader]" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: added peer, starting replication: peer=server-86ebc7ac-66bd-4408-8264-e0cd2136da97 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: added peer, starting replication: peer=server-86ebc7ac-66bd-4408-8264-e0cd2136da97 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: added peer, starting replication: peer=server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: added peer, starting replication: peer=server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" -testLoggerAdapter verbose: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.595Z [WARN] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: rejecting vote request since our last term is greater: candidate=aff24f1d-7182-8979-02c3-856a74358352 last-term=2 last-candidate-term=1 - testing.go:149: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.595Z [WARN] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: rejecting vote request since our last term is greater: candidate=aff24f1d-7182-8979-02c3-856a74358352 last-term=2 last-candidate-term=1 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.595Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.701Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.595624035 +0000 UTC m=+10.925856723 (1 nodes), 54.489083ms from start of poll, 54.816494ms from cluster start. Timeout at 2025-02-09 16:52:16.701634006 +0000 UTC m=+11.031866700, 106.009977ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.701Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.595624035 +0000 UTC m=+10.925856723 (1 nodes), 54.489083ms from start of poll, 54.816494ms from cluster start. Timeout at 2025-02-09 16:52:16.701634006 +0000 UTC m=+11.031866700, 106.009977ms after stability -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.702Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: starting snapshot up to: index=12 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.702Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:16.702Z [INFO] creating new snapshot: path=/tmp/raft3823728204/snapshots/2-12-1739119936702.tmp -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.702Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: snapshot complete up to: index=12 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.702Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:16.703Z [INFO] creating new snapshot: path=/tmp/raft3823728204/snapshots/2-23-1739119936703.tmp -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: copied to local snapshot: bytes=71 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: restored user snapshot: index=1 - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: restored user snapshot: index=1 -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.703Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.709Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.709Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.709Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.709Z [ERROR] server-aff24f1d-7182-8979-02c3-856a74358352: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:16.709Z [INFO] creating new snapshot: path=/tmp/raft3402645027/snapshots/2-23-1739119936709.tmp - testing.go:153: 2025-02-09T16:52:16.709Z [INFO] creating new snapshot: path=/tmp/raft1838337689/snapshots/2-23-1739119936709.tmp -testLoggerAdapter verbose: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.710Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: copied to local snapshot: bytes=71 - testing.go:149: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.710Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: copied to local snapshot: bytes=71 - testing.go:149: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: Installed remote snapshot - testing.go:149: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [INFO] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: Installed remote snapshot -testLoggerAdapter verbose: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [WARN] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: skipping application of old log: index=22 - testing.go:149: server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: 2025-02-09T16:52:16.710Z [WARN] server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef: skipping application of old log: index=22 -testLoggerAdapter verbose: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.710Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: Installed remote snapshot -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.710Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" - testing.go:149: server-86ebc7ac-66bd-4408-8264-e0cd2136da97: 2025-02-09T16:52:16.710Z [INFO] server-86ebc7ac-66bd-4408-8264-e0cd2136da97: Installed remote snapshot - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.710Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.710Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.710Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: pipelining replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.721Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.721Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-2f7f262d-e356-1e68-d5e2-9a64e844f2ef 2f7f262d-e356-1e68-d5e2-9a64e844f2ef}" -testLoggerAdapter verbose: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.721Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" - testing.go:149: server-aff24f1d-7182-8979-02c3-856a74358352: 2025-02-09T16:52:16.721Z [INFO] server-aff24f1d-7182-8979-02c3-856a74358352: aborting pipeline replication: peer="{Voter server-86ebc7ac-66bd-4408-8264-e0cd2136da97 86ebc7ac-66bd-4408-8264-e0cd2136da97}" -testLoggerAdapter verbose: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.722Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" - testing.go:149: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.722Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" -testLoggerAdapter verbose: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.722Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" - testing.go:149: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.722Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.722Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.722Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 Address:581f15ac-8f7d-e9a0-fb62-c8be0df97205} {Suffrage:Voter ID:server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd Address:b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd} {Suffrage:Voter ID:server-8a206b56-6874-9dd2-7ab4-d99682073983 Address:8a206b56-6874-9dd2-7ab4-d99682073983}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.722Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.722Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.722Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: entering follower state: follower="Node at 581f15ac-8f7d-e9a0-fb62-c8be0df97205 [Follower]" leader= - testing.go:149: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.722Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: entering follower state: follower="Node at 581f15ac-8f7d-e9a0-fb62-c8be0df97205 [Follower]" leader= -testLoggerAdapter verbose: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.722Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: entering follower state: follower="Node at b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd [Follower]" leader= - testing.go:149: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.722Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: entering follower state: follower="Node at b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd [Follower]" leader= -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.722Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering follower state: follower="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Follower]" leader= - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.722Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering follower state: follower="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Follower]" leader= -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [WARN] server-8a206b56-6874-9dd2-7ab4-d99682073983: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [WARN] server-8a206b56-6874-9dd2-7ab4-d99682073983: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering candidate state: node="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Candidate]" term=2 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering candidate state: node="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Candidate]" term=2 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: election won: tally=2 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: election won: tally=2 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering leader state: leader="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Leader]" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: entering leader state: leader="Node at 8a206b56-6874-9dd2-7ab4-d99682073983 [Leader]" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: added peer, starting replication: peer=server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: added peer, starting replication: peer=server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: added peer, starting replication: peer=server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: added peer, starting replication: peer=server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.784Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.890Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.784975141 +0000 UTC m=+11.115207829 (1 nodes), 62.126299ms from start of poll, 62.466268ms from cluster start. Timeout at 2025-02-09 16:52:16.890191083 +0000 UTC m=+11.220423771, 105.215942ms after stability - testing.go:149: cluster: 2025-02-09T16:52:16.890Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.784975141 +0000 UTC m=+11.115207829 (1 nodes), 62.126299ms from start of poll, 62.466268ms from cluster start. Timeout at 2025-02-09 16:52:16.890191083 +0000 UTC m=+11.220423771, 105.215942ms after stability -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.890Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: starting snapshot up to: index=12 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.890Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:16.890Z [INFO] creating new snapshot: path=/tmp/raft776709014/snapshots/2-12-1739119936890.tmp -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.891Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: snapshot complete up to: index=12 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.891Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:16.891Z [INFO] creating new snapshot: path=/tmp/raft776709014/snapshots/2-23-1739119936891.tmp -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.891Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: copied to local snapshot: bytes=71 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.891Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: restored user snapshot: index=1 - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: restored user snapshot: index=1 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.893Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.902Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.902Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:16.903Z [INFO] creating new snapshot: path=/tmp/raft3275984814/snapshots/2-23-1739119936903.tmp -testLoggerAdapter verbose: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: copied to local snapshot: bytes=71 - testing.go:149: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: Installed remote snapshot - testing.go:149: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [INFO] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: Installed remote snapshot -testLoggerAdapter verbose: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [WARN] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: skipping application of old log: index=22 - testing.go:149: server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: 2025-02-09T16:52:16.903Z [WARN] server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd: skipping application of old log: index=22 -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.903Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.903Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.903Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.903Z [ERROR] server-8a206b56-6874-9dd2-7ab4-d99682073983: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:16.903Z [INFO] creating new snapshot: path=/tmp/raft1450330207/snapshots/2-23-1739119936903.tmp -testLoggerAdapter verbose: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.904Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: copied to local snapshot: bytes=71 - testing.go:149: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.904Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.904Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: Installed remote snapshot - testing.go:149: server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: 2025-02-09T16:52:16.904Z [INFO] server-581f15ac-8f7d-e9a0-fb62-c8be0df97205: Installed remote snapshot -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.904Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.904Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: pipelining replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.921Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.921Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd b2491ab6-74e7-9dba-cdf1-46b1a85cc0dd}" -testLoggerAdapter verbose: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.921Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" - testing.go:149: server-8a206b56-6874-9dd2-7ab4-d99682073983: 2025-02-09T16:52:16.921Z [INFO] server-8a206b56-6874-9dd2-7ab4-d99682073983: aborting pipeline replication: peer="{Voter server-581f15ac-8f7d-e9a0-fb62-c8be0df97205 581f15ac-8f7d-e9a0-fb62-c8be0df97205}" -testLoggerAdapter verbose: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:16.922Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" - testing.go:149: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:16.922Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.922Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.922Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" -testLoggerAdapter verbose: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:16.922Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" - testing.go:149: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:16.922Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fee11124-1593-3e3d-f3ce-7797aacfb1dd Address:fee11124-1593-3e3d-f3ce-7797aacfb1dd} {Suffrage:Voter ID:server-96b0aae5-9061-8a70-9ec1-37c41708aca1 Address:96b0aae5-9061-8a70-9ec1-37c41708aca1} {Suffrage:Voter ID:server-76394e1b-1313-05db-990d-315968d6709d Address:76394e1b-1313-05db-990d-315968d6709d}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:16.922Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:16.922Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:16.922Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: entering follower state: follower="Node at fee11124-1593-3e3d-f3ce-7797aacfb1dd [Follower]" leader= - testing.go:149: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:16.922Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: entering follower state: follower="Node at fee11124-1593-3e3d-f3ce-7797aacfb1dd [Follower]" leader= -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.922Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering follower state: follower="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Follower]" leader= - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.922Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering follower state: follower="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Follower]" leader= -testLoggerAdapter verbose: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:16.922Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: entering follower state: follower="Node at 76394e1b-1313-05db-990d-315968d6709d [Follower]" leader= - testing.go:149: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:16.922Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: entering follower state: follower="Node at 76394e1b-1313-05db-990d-315968d6709d [Follower]" leader= -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.996Z [WARN] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.996Z [WARN] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering candidate state: node="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Candidate]" term=2 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering candidate state: node="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Candidate]" term=2 -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: election won: tally=2 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: election won: tally=2 -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering leader state: leader="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Leader]" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: entering leader state: leader="Node at 96b0aae5-9061-8a70-9ec1-37c41708aca1 [Leader]" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: added peer, starting replication: peer=server-fee11124-1593-3e3d-f3ce-7797aacfb1dd - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: added peer, starting replication: peer=server-fee11124-1593-3e3d-f3ce-7797aacfb1dd -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: added peer, starting replication: peer=server-76394e1b-1313-05db-990d-315968d6709d - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: added peer, starting replication: peer=server-76394e1b-1313-05db-990d-315968d6709d -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:16.997Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.103Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.997464867 +0000 UTC m=+11.327697555 (1 nodes), 75.135209ms from start of poll, 75.477832ms from cluster start. Timeout at 2025-02-09 16:52:17.103056321 +0000 UTC m=+11.433289009, 105.591454ms after stability - testing.go:149: cluster: 2025-02-09T16:52:17.103Z [INFO] stable state for Leader reached at 2025-02-09 16:52:16.997464867 +0000 UTC m=+11.327697555 (1 nodes), 75.135209ms from start of poll, 75.477832ms from cluster start. Timeout at 2025-02-09 16:52:17.103056321 +0000 UTC m=+11.433289009, 105.591454ms after stability -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.103Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: starting snapshot up to: index=12 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.103Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:17.103Z [INFO] creating new snapshot: path=/tmp/raft3855284576/snapshots/2-12-1739119937103.tmp -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.103Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: snapshot complete up to: index=12 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.103Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:17.104Z [INFO] creating new snapshot: path=/tmp/raft3855284576/snapshots/2-113-1739119937104.tmp -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: copied to local snapshot: bytes=71 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: restored user snapshot: index=1 - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: restored user snapshot: index=1 -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.104Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.113Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.113Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.114Z [INFO] creating new snapshot: path=/tmp/raft2192071279/snapshots/2-113-1739119937114.tmp -testLoggerAdapter verbose: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: copied to local snapshot: bytes=71 - testing.go:149: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: Installed remote snapshot - testing.go:149: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [INFO] server-76394e1b-1313-05db-990d-315968d6709d: Installed remote snapshot -testLoggerAdapter verbose: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [WARN] server-76394e1b-1313-05db-990d-315968d6709d: skipping application of old log: index=22 - testing.go:149: server-76394e1b-1313-05db-990d-315968d6709d: 2025-02-09T16:52:17.114Z [WARN] server-76394e1b-1313-05db-990d-315968d6709d: skipping application of old log: index=22 -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.114Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.114Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.114Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.114Z [ERROR] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.114Z [INFO] creating new snapshot: path=/tmp/raft990092553/snapshots/2-113-1739119937114.tmp -testLoggerAdapter verbose: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:17.115Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: copied to local snapshot: bytes=71 - testing.go:149: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:17.115Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:17.115Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: Installed remote snapshot - testing.go:149: server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: 2025-02-09T16:52:17.115Z [INFO] server-fee11124-1593-3e3d-f3ce-7797aacfb1dd: Installed remote snapshot -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.115Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.115Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: pipelining replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.131Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.131Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-76394e1b-1313-05db-990d-315968d6709d 76394e1b-1313-05db-990d-315968d6709d}" -testLoggerAdapter verbose: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.131Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" - testing.go:149: server-96b0aae5-9061-8a70-9ec1-37c41708aca1: 2025-02-09T16:52:17.131Z [INFO] server-96b0aae5-9061-8a70-9ec1-37c41708aca1: aborting pipeline replication: peer="{Voter server-fee11124-1593-3e3d-f3ce-7797aacfb1dd fee11124-1593-3e3d-f3ce-7797aacfb1dd}" -testLoggerAdapter verbose: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.132Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" - testing.go:149: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.132Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.132Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.132Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" -testLoggerAdapter verbose: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.132Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" - testing.go:149: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.132Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 Address:54982224-49d9-1f98-aeaa-e350b4d4ecd7} {Suffrage:Voter ID:server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d Address:7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d} {Suffrage:Voter ID:server-bf29de11-c0e7-07e1-7152-7d55102847c2 Address:bf29de11-c0e7-07e1-7152-7d55102847c2}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.132Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:17.132Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.132Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: entering follower state: follower="Node at 54982224-49d9-1f98-aeaa-e350b4d4ecd7 [Follower]" leader= - testing.go:149: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.132Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: entering follower state: follower="Node at 54982224-49d9-1f98-aeaa-e350b4d4ecd7 [Follower]" leader= -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.132Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering follower state: follower="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Follower]" leader= - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.132Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering follower state: follower="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Follower]" leader= -testLoggerAdapter verbose: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.132Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: entering follower state: follower="Node at bf29de11-c0e7-07e1-7152-7d55102847c2 [Follower]" leader= - testing.go:149: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.132Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: entering follower state: follower="Node at bf29de11-c0e7-07e1-7152-7d55102847c2 [Follower]" leader= -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [WARN] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [WARN] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering candidate state: node="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Candidate]" term=2 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering candidate state: node="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Candidate]" term=2 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: election won: tally=2 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: election won: tally=2 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering leader state: leader="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Leader]" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: entering leader state: leader="Node at 7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d [Leader]" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: added peer, starting replication: peer=server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: added peer, starting replication: peer=server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: added peer, starting replication: peer=server-bf29de11-c0e7-07e1-7152-7d55102847c2 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: added peer, starting replication: peer=server-bf29de11-c0e7-07e1-7152-7d55102847c2 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.184Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.290Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.184875701 +0000 UTC m=+11.515108389 (1 nodes), 52.103424ms from start of poll, 52.447801ms from cluster start. Timeout at 2025-02-09 16:52:17.290347538 +0000 UTC m=+11.620580226, 105.471837ms after stability - testing.go:149: cluster: 2025-02-09T16:52:17.290Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.184875701 +0000 UTC m=+11.515108389 (1 nodes), 52.103424ms from start of poll, 52.447801ms from cluster start. Timeout at 2025-02-09 16:52:17.290347538 +0000 UTC m=+11.620580226, 105.471837ms after stability -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.290Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: starting snapshot up to: index=12 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.290Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:17.290Z [INFO] creating new snapshot: path=/tmp/raft3163523233/snapshots/2-12-1739119937290.tmp -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: snapshot complete up to: index=12 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:17.291Z [INFO] creating new snapshot: path=/tmp/raft3163523233/snapshots/2-1013-1739119937291.tmp -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: copied to local snapshot: bytes=71 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: restored user snapshot: index=1 - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: restored user snapshot: index=1 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.291Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.292Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.292Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.292Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.292Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.297Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.297Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.297Z [INFO] creating new snapshot: path=/tmp/raft3647749035/snapshots/2-1013-1739119937297.tmp -testLoggerAdapter verbose: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: copied to local snapshot: bytes=71 - testing.go:149: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: Installed remote snapshot - testing.go:149: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [INFO] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: Installed remote snapshot -testLoggerAdapter verbose: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [WARN] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: skipping application of old log: index=22 - testing.go:149: server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: 2025-02-09T16:52:17.298Z [WARN] server-54982224-49d9-1f98-aeaa-e350b4d4ecd7: skipping application of old log: index=22 -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.298Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.298Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.300Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.300Z [ERROR] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.301Z [INFO] creating new snapshot: path=/tmp/raft3597191872/snapshots/2-1013-1739119937301.tmp -testLoggerAdapter verbose: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.301Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: copied to local snapshot: bytes=71 - testing.go:149: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.301Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.301Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: Installed remote snapshot - testing.go:149: server-bf29de11-c0e7-07e1-7152-7d55102847c2: 2025-02-09T16:52:17.301Z [INFO] server-bf29de11-c0e7-07e1-7152-7d55102847c2: Installed remote snapshot -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.301Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.301Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: pipelining replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.315Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.315Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-bf29de11-c0e7-07e1-7152-7d55102847c2 bf29de11-c0e7-07e1-7152-7d55102847c2}" -testLoggerAdapter verbose: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.315Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" - testing.go:149: server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: 2025-02-09T16:52:17.315Z [INFO] server-7207eff5-e4e0-aaa8-bf5d-487b2b8ea00d: aborting pipeline replication: peer="{Voter server-54982224-49d9-1f98-aeaa-e350b4d4ecd7 54982224-49d9-1f98-aeaa-e350b4d4ecd7}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.315Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.315Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" -testLoggerAdapter verbose: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.315Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" - testing.go:149: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.315Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" -testLoggerAdapter verbose: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.316Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" - testing.go:149: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.316Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca Address:2ef2ed47-5a01-1e5d-7708-db0f35e4ddca} {Suffrage:Voter ID:server-1edc9431-a467-130a-87a9-a35d599172b7 Address:1edc9431-a467-130a-87a9-a35d599172b7} {Suffrage:Voter ID:server-2188b6f3-9242-a299-91a8-681ae8da5c2c Address:2188b6f3-9242-a299-91a8-681ae8da5c2c}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.316Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:17.316Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.316Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering follower state: follower="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Follower]" leader= - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.316Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering follower state: follower="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Follower]" leader= -testLoggerAdapter verbose: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.316Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: entering follower state: follower="Node at 1edc9431-a467-130a-87a9-a35d599172b7 [Follower]" leader= - testing.go:149: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.316Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: entering follower state: follower="Node at 1edc9431-a467-130a-87a9-a35d599172b7 [Follower]" leader= -testLoggerAdapter verbose: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.316Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: entering follower state: follower="Node at 2188b6f3-9242-a299-91a8-681ae8da5c2c [Follower]" leader= - testing.go:149: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.316Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: entering follower state: follower="Node at 2188b6f3-9242-a299-91a8-681ae8da5c2c [Follower]" leader= -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [WARN] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [WARN] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering candidate state: node="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Candidate]" term=2 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering candidate state: node="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Candidate]" term=2 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: election won: tally=2 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: election won: tally=2 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering leader state: leader="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Leader]" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.377Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: entering leader state: leader="Node at 2ef2ed47-5a01-1e5d-7708-db0f35e4ddca [Leader]" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: added peer, starting replication: peer=server-1edc9431-a467-130a-87a9-a35d599172b7 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: added peer, starting replication: peer=server-1edc9431-a467-130a-87a9-a35d599172b7 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: added peer, starting replication: peer=server-2188b6f3-9242-a299-91a8-681ae8da5c2c - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: added peer, starting replication: peer=server-2188b6f3-9242-a299-91a8-681ae8da5c2c -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.378Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.483Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.37828519 +0000 UTC m=+11.708517878 (1 nodes), 62.164535ms from start of poll, 62.516419ms from cluster start. Timeout at 2025-02-09 16:52:17.483415198 +0000 UTC m=+11.813647887, 105.130009ms after stability - testing.go:149: cluster: 2025-02-09T16:52:17.483Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.37828519 +0000 UTC m=+11.708517878 (1 nodes), 62.164535ms from start of poll, 62.516419ms from cluster start. Timeout at 2025-02-09 16:52:17.483415198 +0000 UTC m=+11.813647887, 105.130009ms after stability -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.485Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: starting snapshot up to: index=12 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.485Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: starting snapshot up to: index=12 - testing.go:153: 2025-02-09T16:52:17.485Z [INFO] creating new snapshot: path=/tmp/raft3617695562/snapshots/2-12-1739119937485.tmp -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.485Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: snapshot complete up to: index=12 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.485Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: snapshot complete up to: index=12 - testing.go:153: 2025-02-09T16:52:17.486Z [INFO] creating new snapshot: path=/tmp/raft3617695562/snapshots/2-10013-1739119937486.tmp -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.486Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: copied to local snapshot: bytes=71 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.486Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: restored user snapshot: index=1 - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: restored user snapshot: index=1 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.487Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.494Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.494Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.494Z [INFO] creating new snapshot: path=/tmp/raft3463572367/snapshots/2-10013-1739119937494.tmp -testLoggerAdapter verbose: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: copied to local snapshot: bytes=71 - testing.go:149: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: Installed remote snapshot - testing.go:149: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [INFO] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: Installed remote snapshot -testLoggerAdapter verbose: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [WARN] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: skipping application of old log: index=22 - testing.go:149: server-2188b6f3-9242-a299-91a8-681ae8da5c2c: 2025-02-09T16:52:17.494Z [WARN] server-2188b6f3-9242-a299-91a8-681ae8da5c2c: skipping application of old log: index=22 -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.494Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.494Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.497Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.497Z [ERROR] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: failed to get log: index=23 error="log not found" - testing.go:153: 2025-02-09T16:52:17.497Z [INFO] creating new snapshot: path=/tmp/raft711378525/snapshots/2-10013-1739119937497.tmp -testLoggerAdapter verbose: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.497Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: copied to local snapshot: bytes=71 - testing.go:149: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.497Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: copied to local snapshot: bytes=71 -testLoggerAdapter verbose: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.498Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: Installed remote snapshot - testing.go:149: server-1edc9431-a467-130a-87a9-a35d599172b7: 2025-02-09T16:52:17.498Z [INFO] server-1edc9431-a467-130a-87a9-a35d599172b7: Installed remote snapshot -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.498Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.498Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: pipelining replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.517Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.517Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-1edc9431-a467-130a-87a9-a35d599172b7 1edc9431-a467-130a-87a9-a35d599172b7}" -testLoggerAdapter verbose: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.517Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" - testing.go:149: server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: 2025-02-09T16:52:17.517Z [INFO] server-2ef2ed47-5a01-1e5d-7708-db0f35e4ddca: aborting pipeline replication: peer="{Voter server-2188b6f3-9242-a299-91a8-681ae8da5c2c 2188b6f3-9242-a299-91a8-681ae8da5c2c}" ---- PASS: TestRaft_UserRestore (1.19s) +testLoggerAdapter verbose: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.366Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" + testing.go:149: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.366Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" +testLoggerAdapter verbose: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.367Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" + testing.go:149: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.367Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.368Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.368Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 Address:e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09} {Suffrage:Voter ID:server-9785057a-3a69-99de-5328-33b85c34a47b Address:9785057a-3a69-99de-5328-33b85c34a47b} {Suffrage:Voter ID:server-a041f63f-a19b-e910-929e-0eb39f5307ce Address:a041f63f-a19b-e910-929e-0eb39f5307ce}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.368Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:03.368Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.368Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: entering follower state: follower="Node at e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 [Follower]" leader= + testing.go:149: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.368Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: entering follower state: follower="Node at e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 [Follower]" leader= +testLoggerAdapter verbose: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.368Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: entering follower state: follower="Node at 9785057a-3a69-99de-5328-33b85c34a47b [Follower]" leader= + testing.go:149: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.368Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: entering follower state: follower="Node at 9785057a-3a69-99de-5328-33b85c34a47b [Follower]" leader= +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.369Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering follower state: follower="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Follower]" leader= + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.369Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering follower state: follower="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Follower]" leader= +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.457Z [WARN] server-a041f63f-a19b-e910-929e-0eb39f5307ce: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.457Z [WARN] server-a041f63f-a19b-e910-929e-0eb39f5307ce: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering candidate state: node="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Candidate]" term=2 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering candidate state: node="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Candidate]" term=2 +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: election won: tally=2 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: election won: tally=2 +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering leader state: leader="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Leader]" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.458Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: entering leader state: leader="Node at a041f63f-a19b-e910-929e-0eb39f5307ce [Leader]" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: added peer, starting replication: peer=server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: added peer, starting replication: peer=server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: added peer, starting replication: peer=server-9785057a-3a69-99de-5328-33b85c34a47b + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: added peer, starting replication: peer=server-9785057a-3a69-99de-5328-33b85c34a47b +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.459Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.460Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.460Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.566Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.460786511 +0000 UTC m=+11.612580641 (1 nodes), 92.261104ms from start of poll, 111.054932ms from cluster start. Timeout at 2024-01-08 10:38:03.566124384 +0000 UTC m=+11.717918508, 105.337867ms after stability + testing.go:149: cluster: 2024-01-08T10:38:03.566Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.460786511 +0000 UTC m=+11.612580641 (1 nodes), 92.261104ms from start of poll, 111.054932ms from cluster start. Timeout at 2024-01-08 10:38:03.566124384 +0000 UTC m=+11.717918508, 105.337867ms after stability +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.568Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: starting snapshot up to: index=12 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.568Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:03.568Z [INFO] creating new snapshot: path=/tmp/raft4190350935/snapshots/2-12-1704710283568.tmp +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.575Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: snapshot complete up to: index=12 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.575Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:03.590Z [INFO] creating new snapshot: path=/tmp/raft4190350935/snapshots/2-23-1704710283590.tmp +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.593Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: copied to local snapshot: bytes=71 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.593Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: restored user snapshot: index=1 + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: restored user snapshot: index=1 +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.599Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.609Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.609Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.610Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.610Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:03.612Z [INFO] creating new snapshot: path=/tmp/raft289762415/snapshots/2-23-1704710283612.tmp + testing.go:153: 2024-01-08T10:38:03.616Z [INFO] creating new snapshot: path=/tmp/raft3548154031/snapshots/2-23-1704710283616.tmp +testLoggerAdapter verbose: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.622Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: copied to local snapshot: bytes=71 + testing.go:149: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.622Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.623Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: Installed remote snapshot + testing.go:149: server-9785057a-3a69-99de-5328-33b85c34a47b: 2024-01-08T10:38:03.623Z [INFO] server-9785057a-3a69-99de-5328-33b85c34a47b: Installed remote snapshot +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.624Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.624Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" +testLoggerAdapter verbose: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.630Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: copied to local snapshot: bytes=71 + testing.go:149: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.630Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.632Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: Installed remote snapshot + testing.go:149: server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: 2024-01-08T10:38:03.632Z [INFO] server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09: Installed remote snapshot +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.632Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.632Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: pipelining replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.650Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.650Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-9785057a-3a69-99de-5328-33b85c34a47b 9785057a-3a69-99de-5328-33b85c34a47b}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.650Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.650Z [INFO] server-a041f63f-a19b-e910-929e-0eb39f5307ce: aborting pipeline replication: peer="{Voter server-e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09}" +testLoggerAdapter verbose: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.706Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to heartbeat to: peer=e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 error="command timed out" + testing.go:149: server-a041f63f-a19b-e910-929e-0eb39f5307ce: 2024-01-08T10:38:03.706Z [ERROR] server-a041f63f-a19b-e910-929e-0eb39f5307ce: failed to heartbeat to: peer=e844dedc-9ce1-ad19-3f6d-f4c1e24b8d09 error="command timed out" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.715Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.715Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" +testLoggerAdapter verbose: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.716Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" + testing.go:149: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.716Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" +testLoggerAdapter verbose: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.716Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" + testing.go:149: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.716Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3766eda-dd80-744e-1a9d-714ac039ff1e Address:a3766eda-dd80-744e-1a9d-714ac039ff1e} {Suffrage:Voter ID:server-35f1b44b-1f16-6a77-f106-55f2efd188bd Address:35f1b44b-1f16-6a77-f106-55f2efd188bd} {Suffrage:Voter ID:server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d Address:5c438a9b-9e5b-be07-6446-2a2f9ac6762d}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.717Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:03.717Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.717Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering follower state: follower="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Follower]" leader= + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.717Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering follower state: follower="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Follower]" leader= +testLoggerAdapter verbose: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.718Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: entering follower state: follower="Node at 5c438a9b-9e5b-be07-6446-2a2f9ac6762d [Follower]" leader= + testing.go:149: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.718Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: entering follower state: follower="Node at 5c438a9b-9e5b-be07-6446-2a2f9ac6762d [Follower]" leader= +testLoggerAdapter verbose: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.717Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: entering follower state: follower="Node at 35f1b44b-1f16-6a77-f106-55f2efd188bd [Follower]" leader= + testing.go:149: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.717Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: entering follower state: follower="Node at 35f1b44b-1f16-6a77-f106-55f2efd188bd [Follower]" leader= +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [WARN] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [WARN] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering candidate state: node="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Candidate]" term=2 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering candidate state: node="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Candidate]" term=2 +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: election won: tally=2 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.774Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: election won: tally=2 +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering leader state: leader="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Leader]" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: entering leader state: leader="Node at a3766eda-dd80-744e-1a9d-714ac039ff1e [Leader]" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: added peer, starting replication: peer=server-35f1b44b-1f16-6a77-f106-55f2efd188bd + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: added peer, starting replication: peer=server-35f1b44b-1f16-6a77-f106-55f2efd188bd +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: added peer, starting replication: peer=server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.783Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: added peer, starting replication: peer=server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.792Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.792Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.793Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.793Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.899Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.793374913 +0000 UTC m=+11.945169064 (1 nodes), 76.118884ms from start of poll, 78.507867ms from cluster start. Timeout at 2024-01-08 10:38:03.899073842 +0000 UTC m=+12.050868010, 105.698946ms after stability + testing.go:149: cluster: 2024-01-08T10:38:03.899Z [INFO] stable state for Leader reached at 2024-01-08 10:38:03.793374913 +0000 UTC m=+11.945169064 (1 nodes), 76.118884ms from start of poll, 78.507867ms from cluster start. Timeout at 2024-01-08 10:38:03.899073842 +0000 UTC m=+12.050868010, 105.698946ms after stability +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.902Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: starting snapshot up to: index=12 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.902Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:03.902Z [INFO] creating new snapshot: path=/tmp/raft347889942/snapshots/2-12-1704710283902.tmp +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.905Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: snapshot complete up to: index=12 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.905Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:03.907Z [INFO] creating new snapshot: path=/tmp/raft347889942/snapshots/2-23-1704710283907.tmp +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.910Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: copied to local snapshot: bytes=71 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.910Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: restored user snapshot: index=1 + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: restored user snapshot: index=1 +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.911Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.912Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.912Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.912Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.912Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.918Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.918Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.920Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.920Z [ERROR] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:03.922Z [INFO] creating new snapshot: path=/tmp/raft3698323689/snapshots/2-23-1704710283922.tmp +testLoggerAdapter verbose: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.924Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: copied to local snapshot: bytes=71 + testing.go:149: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.924Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.925Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: Installed remote snapshot + testing.go:149: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.925Z [INFO] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: Installed remote snapshot +testLoggerAdapter verbose: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.925Z [WARN] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: skipping application of old log: index=22 + testing.go:149: server-35f1b44b-1f16-6a77-f106-55f2efd188bd: 2024-01-08T10:38:03.925Z [WARN] server-35f1b44b-1f16-6a77-f106-55f2efd188bd: skipping application of old log: index=22 + testing.go:153: 2024-01-08T10:38:03.921Z [INFO] creating new snapshot: path=/tmp/raft33263998/snapshots/2-23-1704710283921.tmp +testLoggerAdapter verbose: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.927Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: copied to local snapshot: bytes=71 + testing.go:149: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.927Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.928Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: Installed remote snapshot + testing.go:149: server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: 2024-01-08T10:38:03.928Z [INFO] server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d: Installed remote snapshot +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.933Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.933Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.934Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.934Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: pipelining replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.966Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.966Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-5c438a9b-9e5b-be07-6446-2a2f9ac6762d 5c438a9b-9e5b-be07-6446-2a2f9ac6762d}" +testLoggerAdapter verbose: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.966Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" + testing.go:149: server-a3766eda-dd80-744e-1a9d-714ac039ff1e: 2024-01-08T10:38:03.966Z [INFO] server-a3766eda-dd80-744e-1a9d-714ac039ff1e: aborting pipeline replication: peer="{Voter server-35f1b44b-1f16-6a77-f106-55f2efd188bd 35f1b44b-1f16-6a77-f106-55f2efd188bd}" +testLoggerAdapter verbose: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:03.975Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" + testing.go:149: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:03.975Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" +testLoggerAdapter verbose: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:03.976Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: entering follower state: follower="Node at f45de3da-83af-17e7-245b-749c49dd5d03 [Follower]" leader= + testing.go:149: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:03.976Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: entering follower state: follower="Node at f45de3da-83af-17e7-245b-749c49dd5d03 [Follower]" leader= +testLoggerAdapter verbose: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:03.976Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" + testing.go:149: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:03.976Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" +testLoggerAdapter verbose: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:03.976Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: entering follower state: follower="Node at 94420a5d-3657-3aa5-48df-ad3c6306faf8 [Follower]" leader= + testing.go:149: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:03.976Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: entering follower state: follower="Node at 94420a5d-3657-3aa5-48df-ad3c6306faf8 [Follower]" leader= +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:03.976Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:03.976Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f45de3da-83af-17e7-245b-749c49dd5d03 Address:f45de3da-83af-17e7-245b-749c49dd5d03} {Suffrage:Voter ID:server-94420a5d-3657-3aa5-48df-ad3c6306faf8 Address:94420a5d-3657-3aa5-48df-ad3c6306faf8} {Suffrage:Voter ID:server-098dcf7a-87ca-f538-a019-1aad945c1a01 Address:098dcf7a-87ca-f538-a019-1aad945c1a01}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:03.977Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:03.977Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:03.977Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering follower state: follower="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Follower]" leader= + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:03.977Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering follower state: follower="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Follower]" leader= +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [WARN] server-098dcf7a-87ca-f538-a019-1aad945c1a01: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [WARN] server-098dcf7a-87ca-f538-a019-1aad945c1a01: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering candidate state: node="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Candidate]" term=2 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering candidate state: node="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Candidate]" term=2 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: election won: tally=2 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.054Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: election won: tally=2 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering leader state: leader="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Leader]" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: entering leader state: leader="Node at 098dcf7a-87ca-f538-a019-1aad945c1a01 [Leader]" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: added peer, starting replication: peer=server-f45de3da-83af-17e7-245b-749c49dd5d03 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: added peer, starting replication: peer=server-f45de3da-83af-17e7-245b-749c49dd5d03 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: added peer, starting replication: peer=server-94420a5d-3657-3aa5-48df-ad3c6306faf8 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: added peer, starting replication: peer=server-94420a5d-3657-3aa5-48df-ad3c6306faf8 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.055Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.056Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.056Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.162Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.056887976 +0000 UTC m=+12.208682123 (1 nodes), 79.618253ms from start of poll, 82.003825ms from cluster start. Timeout at 2024-01-08 10:38:04.162249684 +0000 UTC m=+12.314043808, 105.361685ms after stability + testing.go:149: cluster: 2024-01-08T10:38:04.162Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.056887976 +0000 UTC m=+12.208682123 (1 nodes), 79.618253ms from start of poll, 82.003825ms from cluster start. Timeout at 2024-01-08 10:38:04.162249684 +0000 UTC m=+12.314043808, 105.361685ms after stability +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.164Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: starting snapshot up to: index=12 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.164Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:04.164Z [INFO] creating new snapshot: path=/tmp/raft714474000/snapshots/2-12-1704710284164.tmp +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.166Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: snapshot complete up to: index=12 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.166Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:04.169Z [INFO] creating new snapshot: path=/tmp/raft714474000/snapshots/2-23-1704710284169.tmp +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.171Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: copied to local snapshot: bytes=71 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.171Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: restored user snapshot: index=1 + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: restored user snapshot: index=1 +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.178Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.179Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.179Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.180Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.180Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.181Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.181Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.185Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.185Z [ERROR] server-098dcf7a-87ca-f538-a019-1aad945c1a01: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:04.193Z [INFO] creating new snapshot: path=/tmp/raft1904461371/snapshots/2-23-1704710284190.tmp + testing.go:153: 2024-01-08T10:38:04.195Z [INFO] creating new snapshot: path=/tmp/raft804999286/snapshots/2-23-1704710284195.tmp +testLoggerAdapter verbose: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:04.198Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: copied to local snapshot: bytes=71 + testing.go:149: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:04.198Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:04.198Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: Installed remote snapshot + testing.go:149: server-94420a5d-3657-3aa5-48df-ad3c6306faf8: 2024-01-08T10:38:04.198Z [INFO] server-94420a5d-3657-3aa5-48df-ad3c6306faf8: Installed remote snapshot +testLoggerAdapter verbose: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:04.203Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: copied to local snapshot: bytes=71 + testing.go:149: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:04.203Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:04.204Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: Installed remote snapshot + testing.go:149: server-f45de3da-83af-17e7-245b-749c49dd5d03: 2024-01-08T10:38:04.204Z [INFO] server-f45de3da-83af-17e7-245b-749c49dd5d03: Installed remote snapshot +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.205Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.205Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.206Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.206Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: pipelining replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.234Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.234Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-94420a5d-3657-3aa5-48df-ad3c6306faf8 94420a5d-3657-3aa5-48df-ad3c6306faf8}" +testLoggerAdapter verbose: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.234Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" + testing.go:149: server-098dcf7a-87ca-f538-a019-1aad945c1a01: 2024-01-08T10:38:04.234Z [INFO] server-098dcf7a-87ca-f538-a019-1aad945c1a01: aborting pipeline replication: peer="{Voter server-f45de3da-83af-17e7-245b-749c49dd5d03 f45de3da-83af-17e7-245b-749c49dd5d03}" +testLoggerAdapter verbose: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.239Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" + testing.go:149: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.239Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" +testLoggerAdapter verbose: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.240Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" + testing.go:149: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.240Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.240Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.240Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4986a131-0e29-d8db-84f0-b13d86ecffb0 Address:4986a131-0e29-d8db-84f0-b13d86ecffb0} {Suffrage:Voter ID:server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 Address:fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7} {Suffrage:Voter ID:server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 Address:d00bdb0e-fdd7-ae44-7942-9e3b40b208d6}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.241Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:04.241Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.241Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: entering follower state: follower="Node at 4986a131-0e29-d8db-84f0-b13d86ecffb0 [Follower]" leader= + testing.go:149: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.241Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: entering follower state: follower="Node at 4986a131-0e29-d8db-84f0-b13d86ecffb0 [Follower]" leader= +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.242Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering follower state: follower="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Follower]" leader= + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.242Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering follower state: follower="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Follower]" leader= +testLoggerAdapter verbose: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.254Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: entering follower state: follower="Node at fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 [Follower]" leader= + testing.go:149: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.254Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: entering follower state: follower="Node at fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 [Follower]" leader= +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.306Z [WARN] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.306Z [WARN] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.306Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering candidate state: node="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Candidate]" term=2 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.306Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering candidate state: node="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Candidate]" term=2 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: election won: tally=2 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: election won: tally=2 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering leader state: leader="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Leader]" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: entering leader state: leader="Node at d00bdb0e-fdd7-ae44-7942-9e3b40b208d6 [Leader]" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: added peer, starting replication: peer=server-4986a131-0e29-d8db-84f0-b13d86ecffb0 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: added peer, starting replication: peer=server-4986a131-0e29-d8db-84f0-b13d86ecffb0 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: added peer, starting replication: peer=server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.307Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: added peer, starting replication: peer=server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.308Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.308Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.309Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.309Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.414Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.309642855 +0000 UTC m=+12.461436995 (1 nodes), 68.324811ms from start of poll, 70.65164ms from cluster start. Timeout at 2024-01-08 10:38:04.414695634 +0000 UTC m=+12.566489771, 105.052776ms after stability + testing.go:149: cluster: 2024-01-08T10:38:04.414Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.309642855 +0000 UTC m=+12.461436995 (1 nodes), 68.324811ms from start of poll, 70.65164ms from cluster start. Timeout at 2024-01-08 10:38:04.414695634 +0000 UTC m=+12.566489771, 105.052776ms after stability +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.417Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: starting snapshot up to: index=12 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.417Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:04.417Z [INFO] creating new snapshot: path=/tmp/raft1977575614/snapshots/2-12-1704710284417.tmp +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.420Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: snapshot complete up to: index=12 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.420Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:04.423Z [INFO] creating new snapshot: path=/tmp/raft1977575614/snapshots/2-113-1704710284423.tmp +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.425Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: copied to local snapshot: bytes=71 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.425Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: restored user snapshot: index=1 + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: restored user snapshot: index=1 +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.432Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.433Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.433Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.433Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.433Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.442Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.442Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:04.444Z [INFO] creating new snapshot: path=/tmp/raft1064515515/snapshots/2-113-1704710284444.tmp +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.445Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.445Z [ERROR] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:04.446Z [INFO] creating new snapshot: path=/tmp/raft1617187950/snapshots/2-113-1704710284446.tmp +testLoggerAdapter verbose: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.446Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: copied to local snapshot: bytes=71 + testing.go:149: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.446Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.447Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: Installed remote snapshot + testing.go:149: server-4986a131-0e29-d8db-84f0-b13d86ecffb0: 2024-01-08T10:38:04.447Z [INFO] server-4986a131-0e29-d8db-84f0-b13d86ecffb0: Installed remote snapshot +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.447Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.447Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" +testLoggerAdapter verbose: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.455Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: copied to local snapshot: bytes=71 + testing.go:149: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.455Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.456Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: Installed remote snapshot + testing.go:149: server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: 2024-01-08T10:38:04.456Z [INFO] server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7: Installed remote snapshot +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.457Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.457Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: pipelining replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.479Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.479Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-4986a131-0e29-d8db-84f0-b13d86ecffb0 4986a131-0e29-d8db-84f0-b13d86ecffb0}" +testLoggerAdapter verbose: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.479Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" + testing.go:149: server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: 2024-01-08T10:38:04.479Z [INFO] server-d00bdb0e-fdd7-ae44-7942-9e3b40b208d6: aborting pipeline replication: peer="{Voter server-fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7 fd738e3b-e743-1f65-9a98-2eb7aa1dd6b7}" +testLoggerAdapter verbose: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.486Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" + testing.go:149: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.486Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" +testLoggerAdapter verbose: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.487Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" + testing.go:149: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.487Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.487Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.487Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-feadc6ef-66f3-ec21-4196-056bfb476f4e Address:feadc6ef-66f3-ec21-4196-056bfb476f4e} {Suffrage:Voter ID:server-775f9ab0-931c-56fd-fc34-06b6882983b0 Address:775f9ab0-931c-56fd-fc34-06b6882983b0} {Suffrage:Voter ID:server-0e4f2742-a822-31ba-1a6c-d32ef487226b Address:0e4f2742-a822-31ba-1a6c-d32ef487226b}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.488Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:04.488Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.488Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: entering follower state: follower="Node at feadc6ef-66f3-ec21-4196-056bfb476f4e [Follower]" leader= + testing.go:149: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.488Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: entering follower state: follower="Node at feadc6ef-66f3-ec21-4196-056bfb476f4e [Follower]" leader= +testLoggerAdapter verbose: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.488Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: entering follower state: follower="Node at 775f9ab0-931c-56fd-fc34-06b6882983b0 [Follower]" leader= + testing.go:149: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.488Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: entering follower state: follower="Node at 775f9ab0-931c-56fd-fc34-06b6882983b0 [Follower]" leader= +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.488Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering follower state: follower="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Follower]" leader= + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.488Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering follower state: follower="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Follower]" leader= +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.553Z [WARN] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.553Z [WARN] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.553Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering candidate state: node="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Candidate]" term=2 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.553Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering candidate state: node="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Candidate]" term=2 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: election won: tally=2 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: election won: tally=2 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering leader state: leader="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Leader]" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: entering leader state: leader="Node at 0e4f2742-a822-31ba-1a6c-d32ef487226b [Leader]" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: added peer, starting replication: peer=server-feadc6ef-66f3-ec21-4196-056bfb476f4e + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: added peer, starting replication: peer=server-feadc6ef-66f3-ec21-4196-056bfb476f4e +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: added peer, starting replication: peer=server-775f9ab0-931c-56fd-fc34-06b6882983b0 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: added peer, starting replication: peer=server-775f9ab0-931c-56fd-fc34-06b6882983b0 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.554Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.662Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.554441032 +0000 UTC m=+12.706234883 (1 nodes), 66.177681ms from start of poll, 70.387416ms from cluster start. Timeout at 2024-01-08 10:38:04.661993672 +0000 UTC m=+12.813787791, 107.552908ms after stability + testing.go:149: cluster: 2024-01-08T10:38:04.662Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.554441032 +0000 UTC m=+12.706234883 (1 nodes), 66.177681ms from start of poll, 70.387416ms from cluster start. Timeout at 2024-01-08 10:38:04.661993672 +0000 UTC m=+12.813787791, 107.552908ms after stability +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.665Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: starting snapshot up to: index=12 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.665Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:04.665Z [INFO] creating new snapshot: path=/tmp/raft4035640968/snapshots/2-12-1704710284665.tmp +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.675Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: snapshot complete up to: index=12 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.675Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:04.678Z [INFO] creating new snapshot: path=/tmp/raft4035640968/snapshots/2-1013-1704710284678.tmp +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.690Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: copied to local snapshot: bytes=71 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.690Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.695Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: restored user snapshot: index=1 + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.695Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: restored user snapshot: index=1 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.696Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.696Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.697Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.712Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.712Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.714Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.714Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:04.714Z [INFO] creating new snapshot: path=/tmp/raft3227314528/snapshots/2-1013-1704710284714.tmp +testLoggerAdapter verbose: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.716Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: copied to local snapshot: bytes=71 + testing.go:153: 2024-01-08T10:38:04.716Z [INFO] creating new snapshot: path=/tmp/raft4173808371/snapshots/2-1013-1704710284716.tmp + testing.go:149: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.716Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.718Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: Installed remote snapshot + testing.go:149: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.718Z [INFO] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: Installed remote snapshot +testLoggerAdapter verbose: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.719Z [WARN] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: skipping application of old log: index=22 + testing.go:149: server-feadc6ef-66f3-ec21-4196-056bfb476f4e: 2024-01-08T10:38:04.719Z [WARN] server-feadc6ef-66f3-ec21-4196-056bfb476f4e: skipping application of old log: index=22 +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.719Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.719Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" +testLoggerAdapter verbose: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.727Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: copied to local snapshot: bytes=71 + testing.go:149: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.727Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.728Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: Installed remote snapshot + testing.go:149: server-775f9ab0-931c-56fd-fc34-06b6882983b0: 2024-01-08T10:38:04.728Z [INFO] server-775f9ab0-931c-56fd-fc34-06b6882983b0: Installed remote snapshot +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.729Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.729Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: pipelining replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.758Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.758Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-775f9ab0-931c-56fd-fc34-06b6882983b0 775f9ab0-931c-56fd-fc34-06b6882983b0}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.759Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.759Z [INFO] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: aborting pipeline replication: peer="{Voter server-feadc6ef-66f3-ec21-4196-056bfb476f4e feadc6ef-66f3-ec21-4196-056bfb476f4e}" +testLoggerAdapter verbose: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.810Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to heartbeat to: peer=775f9ab0-931c-56fd-fc34-06b6882983b0 error="command timed out" + testing.go:149: server-0e4f2742-a822-31ba-1a6c-d32ef487226b: 2024-01-08T10:38:04.810Z [ERROR] server-0e4f2742-a822-31ba-1a6c-d32ef487226b: failed to heartbeat to: peer=775f9ab0-931c-56fd-fc34-06b6882983b0 error="command timed out" +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.815Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.815Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.816Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.816Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" +testLoggerAdapter verbose: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:04.816Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" + testing.go:149: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:04.816Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db Address:ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db} {Suffrage:Voter ID:server-3e9c2b68-3865-be7f-e7f4-51c30731865e Address:3e9c2b68-3865-be7f-e7f4-51c30731865e} {Suffrage:Voter ID:server-8d254c8d-6475-2d18-bac1-5407b74eda87 Address:8d254c8d-6475-2d18-bac1-5407b74eda87}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.816Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:04.816Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.816Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering follower state: follower="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Follower]" leader= + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.816Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering follower state: follower="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Follower]" leader= +testLoggerAdapter verbose: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:04.817Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: entering follower state: follower="Node at 8d254c8d-6475-2d18-bac1-5407b74eda87 [Follower]" leader= + testing.go:149: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:04.817Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: entering follower state: follower="Node at 8d254c8d-6475-2d18-bac1-5407b74eda87 [Follower]" leader= +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.817Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: entering follower state: follower="Node at ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db [Follower]" leader= + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.817Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: entering follower state: follower="Node at ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db [Follower]" leader= +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.870Z [WARN] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.870Z [WARN] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.870Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering candidate state: node="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Candidate]" term=2 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.870Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering candidate state: node="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Candidate]" term=2 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: election won: tally=2 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: election won: tally=2 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering leader state: leader="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Leader]" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: entering leader state: leader="Node at 3e9c2b68-3865-be7f-e7f4-51c30731865e [Leader]" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: added peer, starting replication: peer=server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: added peer, starting replication: peer=server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: added peer, starting replication: peer=server-8d254c8d-6475-2d18-bac1-5407b74eda87 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: added peer, starting replication: peer=server-8d254c8d-6475-2d18-bac1-5407b74eda87 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.871Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.874Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.874Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.877Z [WARN] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: rejecting vote request since our last term is greater: candidate=3e9c2b68-3865-be7f-e7f4-51c30731865e last-term=2 last-candidate-term=1 + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:04.877Z [WARN] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: rejecting vote request since our last term is greater: candidate=3e9c2b68-3865-be7f-e7f4-51c30731865e last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:04.985Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.877796802 +0000 UTC m=+13.029590655 (1 nodes), 60.913446ms from start of poll, 63.134868ms from cluster start. Timeout at 2024-01-08 10:38:04.985804701 +0000 UTC m=+13.137598553, 108.007898ms after stability + testing.go:149: cluster: 2024-01-08T10:38:04.985Z [INFO] stable state for Leader reached at 2024-01-08 10:38:04.877796802 +0000 UTC m=+13.029590655 (1 nodes), 60.913446ms from start of poll, 63.134868ms from cluster start. Timeout at 2024-01-08 10:38:04.985804701 +0000 UTC m=+13.137598553, 108.007898ms after stability +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.986Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: starting snapshot up to: index=12 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.986Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: starting snapshot up to: index=12 + testing.go:153: 2024-01-08T10:38:04.986Z [INFO] creating new snapshot: path=/tmp/raft1807093782/snapshots/2-12-1704710284986.tmp +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.986Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: snapshot complete up to: index=12 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.986Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: snapshot complete up to: index=12 + testing.go:153: 2024-01-08T10:38:04.987Z [INFO] creating new snapshot: path=/tmp/raft1807093782/snapshots/2-10013-1704710284987.tmp +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: copied to local snapshot: bytes=71 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: restored user snapshot: index=1 + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: restored user snapshot: index=1 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.987Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.993Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.993Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.996Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:04.996Z [ERROR] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to get log: index=23 error="log not found" + testing.go:153: 2024-01-08T10:38:05.006Z [INFO] creating new snapshot: path=/tmp/raft1386349252/snapshots/2-10013-1704710285006.tmp +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.007Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: copied to local snapshot: bytes=71 + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.007Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.007Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: Installed remote snapshot + testing.go:153: 2024-01-08T10:38:05.007Z [INFO] creating new snapshot: path=/tmp/raft934918510/snapshots/2-10013-1704710285007.tmp + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.007Z [INFO] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: Installed remote snapshot +testLoggerAdapter verbose: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.011Z [WARN] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: skipping application of old log: index=22 + testing.go:149: server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: 2024-01-08T10:38:05.011Z [WARN] server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db: skipping application of old log: index=22 +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.012Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.012Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.043Z [WARN] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to contact: server-id=server-8d254c8d-6475-2d18-bac1-5407b74eda87 time=54.571778ms + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.043Z [WARN] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: failed to contact: server-id=server-8d254c8d-6475-2d18-bac1-5407b74eda87 time=54.571778ms +testLoggerAdapter verbose: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:05.046Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: copied to local snapshot: bytes=71 + testing.go:149: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:05.046Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: copied to local snapshot: bytes=71 +testLoggerAdapter verbose: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:05.047Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: Installed remote snapshot + testing.go:149: server-8d254c8d-6475-2d18-bac1-5407b74eda87: 2024-01-08T10:38:05.047Z [INFO] server-8d254c8d-6475-2d18-bac1-5407b74eda87: Installed remote snapshot +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.047Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.047Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: pipelining replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.098Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.098Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-8d254c8d-6475-2d18-bac1-5407b74eda87 8d254c8d-6475-2d18-bac1-5407b74eda87}" +testLoggerAdapter verbose: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.099Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" + testing.go:149: server-3e9c2b68-3865-be7f-e7f4-51c30731865e: 2024-01-08T10:38:05.099Z [INFO] server-3e9c2b68-3865-be7f-e7f4-51c30731865e: aborting pipeline replication: peer="{Voter server-ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db ff9e13ce-2ef8-5dd4-fedb-e8f70f6a71db}" +--- PASS: TestRaft_UserRestore (1.75s) === RUN TestRaft_SendSnapshotFollower -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.518Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.518Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.518Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.518Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" -testLoggerAdapter verbose: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.518Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" - testing.go:149: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.518Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e Address:6ea491d7-e147-ff1b-4f0b-4c1e2240f79e} {Suffrage:Voter ID:server-6267344d-d02b-48c7-a06d-968b1e08c1d4 Address:6267344d-d02b-48c7-a06d-968b1e08c1d4} {Suffrage:Voter ID:server-4772b2ac-3800-6684-aac5-b7eea888d169 Address:4772b2ac-3800-6684-aac5-b7eea888d169}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.518Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:17.518Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.518Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering follower state: follower="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Follower]" leader= - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.518Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering follower state: follower="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Follower]" leader= -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.518Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: entering follower state: follower="Node at 6267344d-d02b-48c7-a06d-968b1e08c1d4 [Follower]" leader= - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.518Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: entering follower state: follower="Node at 6267344d-d02b-48c7-a06d-968b1e08c1d4 [Follower]" leader= -testLoggerAdapter verbose: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.518Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: entering follower state: follower="Node at 4772b2ac-3800-6684-aac5-b7eea888d169 [Follower]" leader= - testing.go:149: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.518Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: entering follower state: follower="Node at 4772b2ac-3800-6684-aac5-b7eea888d169 [Follower]" leader= -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [WARN] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [WARN] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering candidate state: node="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Candidate]" term=2 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering candidate state: node="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Candidate]" term=2 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: election won: tally=2 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: election won: tally=2 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering leader state: leader="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Leader]" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: entering leader state: leader="Node at 6ea491d7-e147-ff1b-4f0b-4c1e2240f79e [Leader]" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: added peer, starting replication: peer=server-6267344d-d02b-48c7-a06d-968b1e08c1d4 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: added peer, starting replication: peer=server-6267344d-d02b-48c7-a06d-968b1e08c1d4 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: added peer, starting replication: peer=server-4772b2ac-3800-6684-aac5-b7eea888d169 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.603Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: added peer, starting replication: peer=server-4772b2ac-3800-6684-aac5-b7eea888d169 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.604Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.604Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.604Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-4772b2ac-3800-6684-aac5-b7eea888d169 4772b2ac-3800-6684-aac5-b7eea888d169}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.604Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-4772b2ac-3800-6684-aac5-b7eea888d169 4772b2ac-3800-6684-aac5-b7eea888d169}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.709Z [INFO] stable state for Follower reached at 2025-02-09 16:52:17.604003258 +0000 UTC m=+11.934235947 (2 nodes), 85.484355ms from start of poll, 85.83071ms from cluster start. Timeout at 2025-02-09 16:52:17.709235988 +0000 UTC m=+12.039468676, 105.232729ms after stability - testing.go:149: cluster: 2025-02-09T16:52:17.709Z [INFO] stable state for Follower reached at 2025-02-09 16:52:17.604003258 +0000 UTC m=+11.934235947 (2 nodes), 85.484355ms from start of poll, 85.83071ms from cluster start. Timeout at 2025-02-09 16:52:17.709235988 +0000 UTC m=+12.039468676, 105.232729ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.709Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:17.709Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.815Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.709360738 +0000 UTC m=+12.039593423 (1 nodes), 828ns from start of poll, 191.188186ms from cluster start. Timeout at 2025-02-09 16:52:17.815380712 +0000 UTC m=+12.145613400, 106.019977ms after stability - testing.go:149: cluster: 2025-02-09T16:52:17.815Z [INFO] stable state for Leader reached at 2025-02-09 16:52:17.709360738 +0000 UTC m=+12.039593423 (1 nodes), 828ns from start of poll, 191.188186ms from cluster start. Timeout at 2025-02-09 16:52:17.815380712 +0000 UTC m=+12.145613400, 106.019977ms after stability -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to pipeline appendEntries: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" error="append pipeline closed" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to pipeline appendEntries: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" error="append pipeline closed" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to appendEntries to: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" error="failed to connect to peer: 6267344d-d02b-48c7-a06d-968b1e08c1d4" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.815Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to appendEntries to: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" error="failed to connect to peer: 6267344d-d02b-48c7-a06d-968b1e08c1d4" +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.104Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.104Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" +testLoggerAdapter verbose: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.105Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" + testing.go:149: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.105Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.105Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: entering follower state: follower="Node at 50818b78-21b2-f069-74c7-8e8c01d67b7b [Follower]" leader= + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.105Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: entering follower state: follower="Node at 50818b78-21b2-f069-74c7-8e8c01d67b7b [Follower]" leader= +testLoggerAdapter verbose: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.106Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: entering follower state: follower="Node at c3c6dfcf-faaa-d01c-6992-647e765c609e [Follower]" leader= + testing.go:149: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.106Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: entering follower state: follower="Node at c3c6dfcf-faaa-d01c-6992-647e765c609e [Follower]" leader= +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.107Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.107Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-50818b78-21b2-f069-74c7-8e8c01d67b7b Address:50818b78-21b2-f069-74c7-8e8c01d67b7b} {Suffrage:Voter ID:server-c3c6dfcf-faaa-d01c-6992-647e765c609e Address:c3c6dfcf-faaa-d01c-6992-647e765c609e} {Suffrage:Voter ID:server-260b7ece-6832-1529-4e2f-25505b09c99e Address:260b7ece-6832-1529-4e2f-25505b09c99e}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.107Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:05.107Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.107Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering follower state: follower="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Follower]" leader= + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.107Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering follower state: follower="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Follower]" leader= +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.181Z [WARN] server-260b7ece-6832-1529-4e2f-25505b09c99e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.181Z [WARN] server-260b7ece-6832-1529-4e2f-25505b09c99e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.181Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering candidate state: node="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Candidate]" term=2 + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.181Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering candidate state: node="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Candidate]" term=2 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: election won: tally=2 + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: election won: tally=2 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering leader state: leader="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Leader]" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: entering leader state: leader="Node at 260b7ece-6832-1529-4e2f-25505b09c99e [Leader]" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: added peer, starting replication: peer=server-50818b78-21b2-f069-74c7-8e8c01d67b7b + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: added peer, starting replication: peer=server-50818b78-21b2-f069-74c7-8e8c01d67b7b +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: added peer, starting replication: peer=server-c3c6dfcf-faaa-d01c-6992-647e765c609e + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: added peer, starting replication: peer=server-c3c6dfcf-faaa-d01c-6992-647e765c609e +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-c3c6dfcf-faaa-d01c-6992-647e765c609e c3c6dfcf-faaa-d01c-6992-647e765c609e}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-c3c6dfcf-faaa-d01c-6992-647e765c609e c3c6dfcf-faaa-d01c-6992-647e765c609e}" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.182Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.290Z [INFO] stable state for Follower reached at 2024-01-08 10:38:05.182522759 +0000 UTC m=+13.334316610 (2 nodes), 74.879522ms from start of poll, 78.763423ms from cluster start. Timeout at 2024-01-08 10:38:05.290116762 +0000 UTC m=+13.441910875, 107.594265ms after stability + testing.go:149: cluster: 2024-01-08T10:38:05.290Z [INFO] stable state for Follower reached at 2024-01-08 10:38:05.182522759 +0000 UTC m=+13.334316610 (2 nodes), 74.879522ms from start of poll, 78.763423ms from cluster start. Timeout at 2024-01-08 10:38:05.290116762 +0000 UTC m=+13.441910875, 107.594265ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.290Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:05.290Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.434Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.29090896 +0000 UTC m=+13.442703074 (1 nodes), 3.984µs from start of poll, 187.149887ms from cluster start. Timeout at 2024-01-08 10:38:05.433965027 +0000 UTC m=+13.585758877, 143.055803ms after stability + testing.go:149: cluster: 2024-01-08T10:38:05.434Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.29090896 +0000 UTC m=+13.442703074 (1 nodes), 3.984µs from start of poll, 187.149887ms from cluster start. Timeout at 2024-01-08 10:38:05.433965027 +0000 UTC m=+13.585758877, 143.055803ms after stability +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to pipeline appendEntries: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" error="append pipeline closed" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to pipeline appendEntries: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" error="append pipeline closed" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to appendEntries to: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" error="failed to connect to peer: 50818b78-21b2-f069-74c7-8e8c01d67b7b" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.434Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to appendEntries to: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" error="failed to connect to peer: 50818b78-21b2-f069-74c7-8e8c01d67b7b" raft_test.go:1320: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: starting snapshot up to: index=102 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T16:52:17.817Z [INFO] creating new snapshot: path=/tmp/raft833580904/snapshots/2-102-1739119937817.tmp -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: compacting logs: from=1 to=92 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: snapshot complete up to: index=102 - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.817Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.817Z [ERROR] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.817Z [ERROR] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: starting snapshot up to: index=58 - testing.go:149: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: starting snapshot up to: index=58 - testing.go:153: 2025-02-09T16:52:17.817Z [INFO] creating new snapshot: path=/tmp/raft3121326612/snapshots/2-58-1739119937817.tmp -testLoggerAdapter verbose: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: compacting logs: from=1 to=58 - testing.go:149: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: compacting logs: from=1 to=58 -testLoggerAdapter verbose: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: snapshot complete up to: index=58 - testing.go:149: server-4772b2ac-3800-6684-aac5-b7eea888d169: 2025-02-09T16:52:17.817Z [INFO] server-4772b2ac-3800-6684-aac5-b7eea888d169: snapshot complete up to: index=58 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.826Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to get log: index=2 error="log not found" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.826Z [ERROR] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: failed to get log: index=2 error="log not found" - testing.go:153: 2025-02-09T16:52:17.827Z [INFO] creating new snapshot: path=/tmp/raft3835147658/snapshots/2-102-1739119937827.tmp -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: copied to local snapshot: bytes=693 - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: Installed remote snapshot - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [INFO] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: Installed remote snapshot -testLoggerAdapter verbose: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [WARN] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: skipping application of old log: index=102 - testing.go:149: server-6267344d-d02b-48c7-a06d-968b1e08c1d4: 2025-02-09T16:52:17.827Z [WARN] server-6267344d-d02b-48c7-a06d-968b1e08c1d4: skipping application of old log: index=102 -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.827Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.827Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: pipelining replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.828Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-4772b2ac-3800-6684-aac5-b7eea888d169 4772b2ac-3800-6684-aac5-b7eea888d169}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.828Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-4772b2ac-3800-6684-aac5-b7eea888d169 4772b2ac-3800-6684-aac5-b7eea888d169}" -testLoggerAdapter verbose: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.828Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" - testing.go:149: server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: 2025-02-09T16:52:17.828Z [INFO] server-6ea491d7-e147-ff1b-4f0b-4c1e2240f79e: aborting pipeline replication: peer="{Voter server-6267344d-d02b-48c7-a06d-968b1e08c1d4 6267344d-d02b-48c7-a06d-968b1e08c1d4}" ---- PASS: TestRaft_SendSnapshotFollower (0.31s) +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.436Z [ERROR] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.436Z [ERROR] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: starting snapshot up to: index=100 + testing.go:149: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: starting snapshot up to: index=100 + testing.go:153: 2024-01-08T10:38:05.436Z [INFO] creating new snapshot: path=/tmp/raft1515685710/snapshots/2-100-1704710285436.tmp +testLoggerAdapter verbose: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: compacting logs: from=1 to=92 + testing.go:149: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: snapshot complete up to: index=100 + testing.go:149: server-c3c6dfcf-faaa-d01c-6992-647e765c609e: 2024-01-08T10:38:05.436Z [INFO] server-c3c6dfcf-faaa-d01c-6992-647e765c609e: snapshot complete up to: index=100 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.436Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: starting snapshot up to: index=102 + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.436Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T10:38:05.436Z [INFO] creating new snapshot: path=/tmp/raft2647540892/snapshots/2-102-1704710285436.tmp +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.437Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: compacting logs: from=1 to=92 + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.437Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.437Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: snapshot complete up to: index=102 + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.437Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.445Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to get log: index=2 error="log not found" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.445Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to get log: index=2 error="log not found" + testing.go:153: 2024-01-08T10:38:05.446Z [INFO] creating new snapshot: path=/tmp/raft2889670843/snapshots/2-102-1704710285446.tmp +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.446Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: copied to local snapshot: bytes=693 + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.446Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: copied to local snapshot: bytes=693 +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.446Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: Installed remote snapshot + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.446Z [INFO] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: Installed remote snapshot +testLoggerAdapter verbose: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.447Z [WARN] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: skipping application of old log: index=102 + testing.go:149: server-50818b78-21b2-f069-74c7-8e8c01d67b7b: 2024-01-08T10:38:05.447Z [WARN] server-50818b78-21b2-f069-74c7-8e8c01d67b7b: skipping application of old log: index=102 +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: pipelining replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-c3c6dfcf-faaa-d01c-6992-647e765c609e c3c6dfcf-faaa-d01c-6992-647e765c609e}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-c3c6dfcf-faaa-d01c-6992-647e765c609e c3c6dfcf-faaa-d01c-6992-647e765c609e}" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.447Z [INFO] server-260b7ece-6832-1529-4e2f-25505b09c99e: aborting pipeline replication: peer="{Voter server-50818b78-21b2-f069-74c7-8e8c01d67b7b 50818b78-21b2-f069-74c7-8e8c01d67b7b}" +testLoggerAdapter verbose: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.497Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to heartbeat to: peer=50818b78-21b2-f069-74c7-8e8c01d67b7b error="command timed out" + testing.go:149: server-260b7ece-6832-1529-4e2f-25505b09c99e: 2024-01-08T10:38:05.497Z [ERROR] server-260b7ece-6832-1529-4e2f-25505b09c99e: failed to heartbeat to: peer=50818b78-21b2-f069-74c7-8e8c01d67b7b error="command timed out" +--- PASS: TestRaft_SendSnapshotFollower (0.40s) === RUN TestRaft_SendSnapshotAndLogsFollower -testLoggerAdapter verbose: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:17.829Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" - testing.go:149: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:17.829Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" -testLoggerAdapter verbose: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:17.829Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" - testing.go:149: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:17.829Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.829Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.829Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce Address:788eaf0f-d7c0-4bc4-2ebc-38371b1229ce} {Suffrage:Voter ID:server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 Address:bfbab2dc-4e32-aaca-d82e-6c30f5753420} {Suffrage:Voter ID:server-77b14b62-5745-b721-9731-fccc2b92448d Address:77b14b62-5745-b721-9731-fccc2b92448d}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:17.829Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:17.829Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:17.829Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: entering follower state: follower="Node at 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce [Follower]" leader= - testing.go:149: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:17.829Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: entering follower state: follower="Node at 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce [Follower]" leader= -testLoggerAdapter verbose: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:17.829Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: entering follower state: follower="Node at bfbab2dc-4e32-aaca-d82e-6c30f5753420 [Follower]" leader= - testing.go:149: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:17.829Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: entering follower state: follower="Node at bfbab2dc-4e32-aaca-d82e-6c30f5753420 [Follower]" leader= -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.830Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering follower state: follower="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Follower]" leader= - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.830Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering follower state: follower="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Follower]" leader= -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [WARN] server-77b14b62-5745-b721-9731-fccc2b92448d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [WARN] server-77b14b62-5745-b721-9731-fccc2b92448d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering candidate state: node="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Candidate]" term=2 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering candidate state: node="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Candidate]" term=2 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: election won: tally=2 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: election won: tally=2 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering leader state: leader="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Leader]" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: entering leader state: leader="Node at 77b14b62-5745-b721-9731-fccc2b92448d [Leader]" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: added peer, starting replication: peer=server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: added peer, starting replication: peer=server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: added peer, starting replication: peer=server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: added peer, starting replication: peer=server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 bfbab2dc-4e32-aaca-d82e-6c30f5753420}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 bfbab2dc-4e32-aaca-d82e-6c30f5753420}" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:17.901Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.007Z [INFO] stable state for Follower reached at 2025-02-09 16:52:17.901943991 +0000 UTC m=+12.232176679 (2 nodes), 72.044533ms from start of poll, 72.399985ms from cluster start. Timeout at 2025-02-09 16:52:18.007606888 +0000 UTC m=+12.337839576, 105.662897ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.007Z [INFO] stable state for Follower reached at 2025-02-09 16:52:17.901943991 +0000 UTC m=+12.232176679 (2 nodes), 72.044533ms from start of poll, 72.399985ms from cluster start. Timeout at 2025-02-09 16:52:18.007606888 +0000 UTC m=+12.337839576, 105.662897ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.007Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:18.007Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.113Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.007723312 +0000 UTC m=+12.337955998 (1 nodes), 766ns from start of poll, 178.179304ms from cluster start. Timeout at 2025-02-09 16:52:18.113534547 +0000 UTC m=+12.443767233, 105.811235ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.113Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.007723312 +0000 UTC m=+12.337955998 (1 nodes), 766ns from start of poll, 178.179304ms from cluster start. Timeout at 2025-02-09 16:52:18.113534547 +0000 UTC m=+12.443767233, 105.811235ms after stability -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to pipeline appendEntries: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" error="append pipeline closed" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to pipeline appendEntries: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" error="append pipeline closed" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to appendEntries to: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" error="failed to connect to peer: 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.113Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to appendEntries to: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" error="failed to connect to peer: 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.507Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.507Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.507Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.507Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" +testLoggerAdapter verbose: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.508Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" + testing.go:149: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.508Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-cd998d89-f625-c93a-5456-57496f6a21e7 Address:cd998d89-f625-c93a-5456-57496f6a21e7} {Suffrage:Voter ID:server-6b413bb1-5a87-f14d-40cb-58dabc00e143 Address:6b413bb1-5a87-f14d-40cb-58dabc00e143} {Suffrage:Voter ID:server-7cc1abf9-3541-33cc-2557-1e4612c59148 Address:7cc1abf9-3541-33cc-2557-1e4612c59148}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.508Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:05.508Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.508Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering follower state: follower="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Follower]" leader= + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.508Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering follower state: follower="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Follower]" leader= +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.509Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: entering follower state: follower="Node at 6b413bb1-5a87-f14d-40cb-58dabc00e143 [Follower]" leader= + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.509Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: entering follower state: follower="Node at 6b413bb1-5a87-f14d-40cb-58dabc00e143 [Follower]" leader= +testLoggerAdapter verbose: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.509Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: entering follower state: follower="Node at 7cc1abf9-3541-33cc-2557-1e4612c59148 [Follower]" leader= + testing.go:149: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.509Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: entering follower state: follower="Node at 7cc1abf9-3541-33cc-2557-1e4612c59148 [Follower]" leader= +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.577Z [WARN] server-cd998d89-f625-c93a-5456-57496f6a21e7: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.577Z [WARN] server-cd998d89-f625-c93a-5456-57496f6a21e7: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.577Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering candidate state: node="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Candidate]" term=2 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.577Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering candidate state: node="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Candidate]" term=2 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: election won: tally=2 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: election won: tally=2 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering leader state: leader="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Leader]" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: entering leader state: leader="Node at cd998d89-f625-c93a-5456-57496f6a21e7 [Leader]" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: added peer, starting replication: peer=server-6b413bb1-5a87-f14d-40cb-58dabc00e143 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: added peer, starting replication: peer=server-6b413bb1-5a87-f14d-40cb-58dabc00e143 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: added peer, starting replication: peer=server-7cc1abf9-3541-33cc-2557-1e4612c59148 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: added peer, starting replication: peer=server-7cc1abf9-3541-33cc-2557-1e4612c59148 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-7cc1abf9-3541-33cc-2557-1e4612c59148 7cc1abf9-3541-33cc-2557-1e4612c59148}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-7cc1abf9-3541-33cc-2557-1e4612c59148 7cc1abf9-3541-33cc-2557-1e4612c59148}" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.578Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.578Z [WARN] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: rejecting vote request since our last term is greater: candidate=cd998d89-f625-c93a-5456-57496f6a21e7 last-term=2 last-candidate-term=1 + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.578Z [WARN] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: rejecting vote request since our last term is greater: candidate=cd998d89-f625-c93a-5456-57496f6a21e7 last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.686Z [INFO] stable state for Follower reached at 2024-01-08 10:38:05.579630867 +0000 UTC m=+13.731425015 (2 nodes), 70.809181ms from start of poll, 73.183863ms from cluster start. Timeout at 2024-01-08 10:38:05.685933285 +0000 UTC m=+13.837727401, 106.302386ms after stability + testing.go:149: cluster: 2024-01-08T10:38:05.686Z [INFO] stable state for Follower reached at 2024-01-08 10:38:05.579630867 +0000 UTC m=+13.731425015 (2 nodes), 70.809181ms from start of poll, 73.183863ms from cluster start. Timeout at 2024-01-08 10:38:05.685933285 +0000 UTC m=+13.837727401, 106.302386ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.686Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:05.686Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.793Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.686548919 +0000 UTC m=+13.838343047 (1 nodes), 3.578µs from start of poll, 180.101895ms from cluster start. Timeout at 2024-01-08 10:38:05.793814568 +0000 UTC m=+13.945608416, 107.265369ms after stability + testing.go:149: cluster: 2024-01-08T10:38:05.793Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.686548919 +0000 UTC m=+13.838343047 (1 nodes), 3.578µs from start of poll, 180.101895ms from cluster start. Timeout at 2024-01-08 10:38:05.793814568 +0000 UTC m=+13.945608416, 107.265369ms after stability +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to pipeline appendEntries: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="append pipeline closed" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to pipeline appendEntries: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="append pipeline closed" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to appendEntries to: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.794Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to appendEntries to: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.808Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to appendEntries to: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.808Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to appendEntries to: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.810Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to heartbeat to: peer=6b413bb1-5a87-f14d-40cb-58dabc00e143 error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.810Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to heartbeat to: peer=6b413bb1-5a87-f14d-40cb-58dabc00e143 error="failed to connect to peer: 6b413bb1-5a87-f14d-40cb-58dabc00e143" raft_test.go:1362: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.115Z [ERROR] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: failed to take snapshot: error="nothing new to snapshot" - testing.go:149: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.115Z [ERROR] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: failed to take snapshot: error="nothing new to snapshot" -testLoggerAdapter verbose: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: starting snapshot up to: index=98 - testing.go:149: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: starting snapshot up to: index=98 - testing.go:153: 2025-02-09T16:52:18.115Z [INFO] creating new snapshot: path=/tmp/raft777594996/snapshots/2-98-1739119938115.tmp -testLoggerAdapter verbose: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: compacting logs: from=1 to=92 - testing.go:149: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: snapshot complete up to: index=98 - testing.go:149: server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: 2025-02-09T16:52:18.115Z [INFO] server-bfbab2dc-4e32-aaca-d82e-6c30f5753420: snapshot complete up to: index=98 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.115Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: starting snapshot up to: index=102 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.115Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: starting snapshot up to: index=102 - testing.go:153: 2025-02-09T16:52:18.115Z [INFO] creating new snapshot: path=/tmp/raft317753678/snapshots/2-102-1739119938115.tmp -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: compacting logs: from=1 to=92 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: compacting logs: from=1 to=92 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: snapshot complete up to: index=102 - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: snapshot complete up to: index=102 -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to heartbeat to: peer=788eaf0f-d7c0-4bc4-2ebc-38371b1229ce error="failed to connect to peer: 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.116Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to heartbeat to: peer=788eaf0f-d7c0-4bc4-2ebc-38371b1229ce error="failed to connect to peer: 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: starting snapshot up to: index=102 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: starting snapshot up to: index=102 + testing.go:153: 2024-01-08T10:38:05.814Z [INFO] creating new snapshot: path=/tmp/raft2395712373/snapshots/2-102-1704710285814.tmp +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: compacting logs: from=1 to=92 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: compacting logs: from=1 to=92 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: snapshot complete up to: index=102 + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.814Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: snapshot complete up to: index=102 +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.814Z [ERROR] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: failed to take snapshot: error="nothing new to snapshot" + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.814Z [ERROR] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: failed to take snapshot: error="nothing new to snapshot" +testLoggerAdapter verbose: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: starting snapshot up to: index=68 + testing.go:149: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: starting snapshot up to: index=68 + testing.go:153: 2024-01-08T10:38:05.815Z [INFO] creating new snapshot: path=/tmp/raft3810678984/snapshots/2-68-1704710285815.tmp +testLoggerAdapter verbose: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: compacting logs: from=1 to=68 + testing.go:149: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: compacting logs: from=1 to=68 +testLoggerAdapter verbose: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: snapshot complete up to: index=68 + testing.go:149: server-7cc1abf9-3541-33cc-2557-1e4612c59148: 2024-01-08T10:38:05.815Z [INFO] server-7cc1abf9-3541-33cc-2557-1e4612c59148: snapshot complete up to: index=68 +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.822Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to get log: index=2 error="log not found" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.822Z [ERROR] server-cd998d89-f625-c93a-5456-57496f6a21e7: failed to get log: index=2 error="log not found" raft_test.go:1383: [INFO] Finished apply without behind follower -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.123Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to get log: index=2 error="log not found" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.123Z [ERROR] server-77b14b62-5745-b721-9731-fccc2b92448d: failed to get log: index=2 error="log not found" - testing.go:153: 2025-02-09T16:52:18.124Z [INFO] creating new snapshot: path=/tmp/raft1536789051/snapshots/2-102-1739119938124.tmp -testLoggerAdapter verbose: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.124Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: copied to local snapshot: bytes=693 - testing.go:149: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.124Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: copied to local snapshot: bytes=693 -testLoggerAdapter verbose: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.124Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: Installed remote snapshot - testing.go:149: server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: 2025-02-09T16:52:18.124Z [INFO] server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce: Installed remote snapshot -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.124Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.124Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: pipelining replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.130Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 bfbab2dc-4e32-aaca-d82e-6c30f5753420}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.130Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-bfbab2dc-4e32-aaca-d82e-6c30f5753420 bfbab2dc-4e32-aaca-d82e-6c30f5753420}" -testLoggerAdapter verbose: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.130Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" - testing.go:149: server-77b14b62-5745-b721-9731-fccc2b92448d: 2025-02-09T16:52:18.130Z [INFO] server-77b14b62-5745-b721-9731-fccc2b92448d: aborting pipeline replication: peer="{Voter server-788eaf0f-d7c0-4bc4-2ebc-38371b1229ce 788eaf0f-d7c0-4bc4-2ebc-38371b1229ce}" ---- PASS: TestRaft_SendSnapshotAndLogsFollower (0.30s) + testing.go:153: 2024-01-08T10:38:05.823Z [INFO] creating new snapshot: path=/tmp/raft3679848351/snapshots/2-102-1704710285823.tmp +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.823Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: copied to local snapshot: bytes=693 + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.823Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: copied to local snapshot: bytes=693 +testLoggerAdapter verbose: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.824Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: Installed remote snapshot + testing.go:149: server-6b413bb1-5a87-f14d-40cb-58dabc00e143: 2024-01-08T10:38:05.824Z [INFO] server-6b413bb1-5a87-f14d-40cb-58dabc00e143: Installed remote snapshot +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.824Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.824Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: pipelining replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.845Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-7cc1abf9-3541-33cc-2557-1e4612c59148 7cc1abf9-3541-33cc-2557-1e4612c59148}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.845Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-7cc1abf9-3541-33cc-2557-1e4612c59148 7cc1abf9-3541-33cc-2557-1e4612c59148}" +testLoggerAdapter verbose: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.846Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" + testing.go:149: server-cd998d89-f625-c93a-5456-57496f6a21e7: 2024-01-08T10:38:05.846Z [INFO] server-cd998d89-f625-c93a-5456-57496f6a21e7: aborting pipeline replication: peer="{Voter server-6b413bb1-5a87-f14d-40cb-58dabc00e143 6b413bb1-5a87-f14d-40cb-58dabc00e143}" +--- PASS: TestRaft_SendSnapshotAndLogsFollower (0.35s) === RUN TestRaft_ReJoinFollower -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.131Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.131Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.131Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.131Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" -testLoggerAdapter verbose: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.131Z [INFO] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" - testing.go:149: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.131Z [INFO] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061} {Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.131Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:18.131Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.131Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering follower state: follower="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Follower]" leader= - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.131Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering follower state: follower="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Follower]" leader= -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.131Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering follower state: follower="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Follower]" leader= - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.131Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering follower state: follower="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Follower]" leader= -testLoggerAdapter verbose: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.131Z [INFO] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: entering follower state: follower="Node at 6f58c0d3-2b07-0d32-0532-f7a56763bca9 [Follower]" leader= - testing.go:149: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.131Z [INFO] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: entering follower state: follower="Node at 6f58c0d3-2b07-0d32-0532-f7a56763bca9 [Follower]" leader= -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.207Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.207Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.207Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering candidate state: node="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Candidate]" term=2 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.207Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering candidate state: node="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Candidate]" term=2 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: election won: tally=2 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: election won: tally=2 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering leader state: leader="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Leader]" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering leader state: leader="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Leader]" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.208Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.313Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.208389713 +0000 UTC m=+12.538622401 (1 nodes), 76.894807ms from start of poll, 77.227731ms from cluster start. Timeout at 2025-02-09 16:52:18.313474802 +0000 UTC m=+12.643707488, 105.085087ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.313Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.208389713 +0000 UTC m=+12.538622401 (1 nodes), 76.894807ms from start of poll, 77.227731ms from cluster start. Timeout at 2025-02-09 16:52:18.313474802 +0000 UTC m=+12.643707488, 105.085087ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.319Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:18.319Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.424Z [INFO] stable state for Follower reached at 2025-02-09 16:52:18.319080409 +0000 UTC m=+12.649313095 (2 nodes), 1.262µs from start of poll, 187.918425ms from cluster start. Timeout at 2025-02-09 16:52:18.424441751 +0000 UTC m=+12.754674439, 105.361344ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.424Z [INFO] stable state for Follower reached at 2025-02-09 16:52:18.319080409 +0000 UTC m=+12.649313095 (2 nodes), 1.262µs from start of poll, 187.918425ms from cluster start. Timeout at 2025-02-09 16:52:18.424441751 +0000 UTC m=+12.754674439, 105.361344ms after stability -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: updating configuration: command=RemoveServer server-id=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 server-addr= servers="[{Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: updating configuration: command=RemoveServer server-id=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 server-addr= servers="[{Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9}]" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: removed peer, stopping replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 last-index=3 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: removed peer, stopping replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 last-index=3 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.424Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.512Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: heartbeat timeout reached, starting election: last-leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.512Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: heartbeat timeout reached, starting election: last-leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.512Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering candidate state: node="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Candidate]" term=3 - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.512Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering candidate state: node="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Candidate]" term=3 -testLoggerAdapter verbose: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.512Z [WARN] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d - testing.go:149: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.512Z [WARN] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.512Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.512Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: updating configuration: command=AddStaging server-id=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 server-addr=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 servers="[{Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9} {Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}]" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: updating configuration: command=AddStaging server-id=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 server-addr=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 servers="[{Suffrage:Voter ID:server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d Address:a25c7164-4322-7d4c-2d44-76f5c6b8b64d} {Suffrage:Voter ID:server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 Address:6f58c0d3-2b07-0d32-0532-f7a56763bca9} {Suffrage:Voter ID:server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 Address:7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}]" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [ERROR] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: peer has newer term, stopping replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [ERROR] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: peer has newer term, stopping replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering follower state: follower="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Follower]" leader= - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering follower state: follower="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.530Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:18.530Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.530Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.588Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: Election timeout reached, restarting election - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.588Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: Election timeout reached, restarting election -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.588Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering candidate state: node="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Candidate]" term=4 - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.588Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering candidate state: node="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Candidate]" term=4 -testLoggerAdapter verbose: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.589Z [WARN] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d - testing.go:149: server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: 2025-02-09T16:52:18.589Z [WARN] server-6f58c0d3-2b07-0d32-0532-f7a56763bca9: rejecting vote request since we have a leader: from=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 leader=a25c7164-4322-7d4c-2d44-76f5c6b8b64d -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.589Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: rejecting vote request since our last index is greater: candidate=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 last-index=4 last-candidate-index=3 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.589Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: rejecting vote request since our last index is greater: candidate=7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 last-index=4 last-candidate-index=3 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering candidate state: node="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Candidate]" term=5 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering candidate state: node="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Candidate]" term=5 -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.599Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering follower state: follower="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Follower]" leader= - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.599Z [INFO] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: entering follower state: follower="Node at 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 [Follower]" leader= -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: election won: tally=2 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: election won: tally=2 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering leader state: leader="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Leader]" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: entering leader state: leader="Node at a25c7164-4322-7d4c-2d44-76f5c6b8b64d [Leader]" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: added peer, starting replication: peer=server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 -testLoggerAdapter verbose: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.599Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: failed to get previous log: previous-index=4 last-index=3 error="log not found" - testing.go:149: server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: 2025-02-09T16:52:18.599Z [WARN] server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061: failed to get previous log: previous-index=4 last-index=3 error="log not found" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: appendEntries rejected, sending older logs: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" next=4 - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [WARN] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: appendEntries rejected, sending older logs: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" next=4 -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.599Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: pipelining replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.705Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.599923161 +0000 UTC m=+12.930155849 (1 nodes), 69.347484ms from start of poll, 468.761179ms from cluster start. Timeout at 2025-02-09 16:52:18.70588223 +0000 UTC m=+13.036114918, 105.959069ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.705Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.599923161 +0000 UTC m=+12.930155849 (1 nodes), 69.347484ms from start of poll, 468.761179ms from cluster start. Timeout at 2025-02-09 16:52:18.70588223 +0000 UTC m=+13.036114918, 105.959069ms after stability -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.706Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.706Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-7d3dfcf7-db1c-c8ec-517d-b58b8ecca061 7d3dfcf7-db1c-c8ec-517d-b58b8ecca061}" -testLoggerAdapter verbose: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.706Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" - testing.go:149: server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: 2025-02-09T16:52:18.706Z [INFO] server-a25c7164-4322-7d4c-2d44-76f5c6b8b64d: aborting pipeline replication: peer="{Voter server-6f58c0d3-2b07-0d32-0532-f7a56763bca9 6f58c0d3-2b07-0d32-0532-f7a56763bca9}" ---- PASS: TestRaft_ReJoinFollower (0.58s) +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.847Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.847Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.847Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.847Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.847Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.847Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:05.847Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:05.847Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.847Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering follower state: follower="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Follower]" leader= + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.847Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering follower state: follower="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Follower]" leader= +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.847Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader= + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.847Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader= +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.847Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering follower state: follower="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Follower]" leader= + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.847Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering follower state: follower="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Follower]" leader= +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.913Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.913Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.913Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering candidate state: node="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Candidate]" term=2 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.913Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering candidate state: node="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Candidate]" term=2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: election won: tally=2 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: election won: tally=2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering leader state: leader="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Leader]" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering leader state: leader="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Leader]" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-152a1577-3c8d-5394-90a1-61403f0ab0a2 + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=2 +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: duplicate requestVote for same term: term=2 + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:05.914Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-152a1577-3c8d-5394-90a1-61403f0ab0a2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.915Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.915Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.915Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:05.915Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.914Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.916Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:05.916Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.022Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.916776693 +0000 UTC m=+14.068570831 (1 nodes), 69.282592ms from start of poll, 69.719808ms from cluster start. Timeout at 2024-01-08 10:38:06.022283973 +0000 UTC m=+14.174078121, 105.50729ms after stability + testing.go:149: cluster: 2024-01-08T10:38:06.022Z [INFO] stable state for Leader reached at 2024-01-08 10:38:05.916776693 +0000 UTC m=+14.068570831 (1 nodes), 69.282592ms from start of poll, 69.719808ms from cluster start. Timeout at 2024-01-08 10:38:06.022283973 +0000 UTC m=+14.174078121, 105.50729ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.029Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:06.029Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.142Z [INFO] stable state for Follower reached at 2024-01-08 10:38:06.030107391 +0000 UTC m=+14.181901519 (2 nodes), 4.564µs from start of poll, 183.050496ms from cluster start. Timeout at 2024-01-08 10:38:06.141883097 +0000 UTC m=+14.293677223, 111.775704ms after stability + testing.go:149: cluster: 2024-01-08T10:38:06.142Z [INFO] stable state for Follower reached at 2024-01-08 10:38:06.030107391 +0000 UTC m=+14.181901519 (2 nodes), 4.564µs from start of poll, 183.050496ms from cluster start. Timeout at 2024-01-08 10:38:06.141883097 +0000 UTC m=+14.293677223, 111.775704ms after stability +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.142Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: updating configuration: command=RemoveServer server-id=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 server-addr= servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.142Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: updating configuration: command=RemoveServer server-id=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 server-addr= servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2}]" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.143Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: removed peer, stopping replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 last-index=3 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.143Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: removed peer, stopping replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 last-index=3 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.143Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.143Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.201Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: heartbeat timeout reached, starting election: last-leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.201Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: heartbeat timeout reached, starting election: last-leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.201Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=3 + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.201Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=3 +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.202Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.202Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.202Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.202Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.249Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: updating configuration: command=AddStaging server-id=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 server-addr=17d6bb27-8a9d-7e78-e2e5-f808df48d144 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144}]" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.249Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: updating configuration: command=AddStaging server-id=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 server-addr=17d6bb27-8a9d-7e78-e2e5-f808df48d144 servers="[{Suffrage:Voter ID:server-c834ac43-aa1a-f77a-9f09-22589df94c1e Address:c834ac43-aa1a-f77a-9f09-22589df94c1e} {Suffrage:Voter ID:server-152a1577-3c8d-5394-90a1-61403f0ab0a2 Address:152a1577-3c8d-5394-90a1-61403f0ab0a2} {Suffrage:Voter ID:server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 Address:17d6bb27-8a9d-7e78-e2e5-f808df48d144}]" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [ERROR] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: peer has newer term, stopping replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [ERROR] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: peer has newer term, stopping replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering follower state: follower="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Follower]" leader= + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering follower state: follower="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.250Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:06.250Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.250Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.265Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: Election timeout reached, restarting election + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.265Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: Election timeout reached, restarting election +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.265Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=4 + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.265Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering candidate state: node="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Candidate]" term=4 +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.266Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.266Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: rejecting vote request since we have a leader: from=17d6bb27-8a9d-7e78-e2e5-f808df48d144 leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.266Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since our last index is greater: candidate=17d6bb27-8a9d-7e78-e2e5-f808df48d144 last-index=4 last-candidate-index=3 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.266Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: rejecting vote request since our last index is greater: candidate=17d6bb27-8a9d-7e78-e2e5-f808df48d144 last-index=4 last-candidate-index=3 +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.325Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: heartbeat timeout reached, starting election: last-leader=c834ac43-aa1a-f77a-9f09-22589df94c1e + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.325Z [WARN] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: heartbeat timeout reached, starting election: last-leader=c834ac43-aa1a-f77a-9f09-22589df94c1e +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.325Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering candidate state: node="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Candidate]" term=3 + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.325Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering candidate state: node="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Candidate]" term=3 +testLoggerAdapter verbose: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.326Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering follower state: follower="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Follower]" leader= + testing.go:149: server-152a1577-3c8d-5394-90a1-61403f0ab0a2: 2024-01-08T10:38:06.326Z [INFO] server-152a1577-3c8d-5394-90a1-61403f0ab0a2: entering follower state: follower="Node at 152a1577-3c8d-5394-90a1-61403f0ab0a2 [Follower]" leader= +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.329Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.329Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.329Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering candidate state: node="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Candidate]" term=5 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.329Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering candidate state: node="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Candidate]" term=5 +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.330Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader= + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.330Z [INFO] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: entering follower state: follower="Node at 17d6bb27-8a9d-7e78-e2e5-f808df48d144 [Follower]" leader= +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: election won: tally=2 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: election won: tally=2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering leader state: leader="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Leader]" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: entering leader state: leader="Node at c834ac43-aa1a-f77a-9f09-22589df94c1e [Leader]" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-152a1577-3c8d-5394-90a1-61403f0ab0a2 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-152a1577-3c8d-5394-90a1-61403f0ab0a2 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: added peer, starting replication: peer=server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 +testLoggerAdapter verbose: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.330Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: failed to get previous log: previous-index=4 last-index=3 error="log not found" + testing.go:149: server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: 2024-01-08T10:38:06.330Z [WARN] server-17d6bb27-8a9d-7e78-e2e5-f808df48d144: failed to get previous log: previous-index=4 last-index=3 error="log not found" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: appendEntries rejected, sending older logs: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" next=4 + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [WARN] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: appendEntries rejected, sending older logs: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" next=4 +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.330Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: pipelining replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.437Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.330523313 +0000 UTC m=+14.482317164 (1 nodes), 80.174364ms from start of poll, 483.466141ms from cluster start. Timeout at 2024-01-08 10:38:06.437805611 +0000 UTC m=+14.589599462, 107.282298ms after stability + testing.go:149: cluster: 2024-01-08T10:38:06.437Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.330523313 +0000 UTC m=+14.482317164 (1 nodes), 80.174364ms from start of poll, 483.466141ms from cluster start. Timeout at 2024-01-08 10:38:06.437805611 +0000 UTC m=+14.589599462, 107.282298ms after stability +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.438Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.438Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-17d6bb27-8a9d-7e78-e2e5-f808df48d144 17d6bb27-8a9d-7e78-e2e5-f808df48d144}" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.438Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.438Z [INFO] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: aborting pipeline replication: peer="{Voter server-152a1577-3c8d-5394-90a1-61403f0ab0a2 152a1577-3c8d-5394-90a1-61403f0ab0a2}" +testLoggerAdapter verbose: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.489Z [ERROR] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: failed to heartbeat to: peer=152a1577-3c8d-5394-90a1-61403f0ab0a2 error="command timed out" + testing.go:149: server-c834ac43-aa1a-f77a-9f09-22589df94c1e: 2024-01-08T10:38:06.489Z [ERROR] server-c834ac43-aa1a-f77a-9f09-22589df94c1e: failed to heartbeat to: peer=152a1577-3c8d-5394-90a1-61403f0ab0a2 error="command timed out" +--- PASS: TestRaft_ReJoinFollower (0.64s) === RUN TestRaft_LeaderLeaseExpire -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.706Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c722b287-10c4-78d0-012e-bbd1685468ba Address:c722b287-10c4-78d0-012e-bbd1685468ba} {Suffrage:Voter ID:server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b Address:2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}]" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.706Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c722b287-10c4-78d0-012e-bbd1685468ba Address:c722b287-10c4-78d0-012e-bbd1685468ba} {Suffrage:Voter ID:server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b Address:2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}]" -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:18.706Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c722b287-10c4-78d0-012e-bbd1685468ba Address:c722b287-10c4-78d0-012e-bbd1685468ba} {Suffrage:Voter ID:server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b Address:2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}]" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:18.706Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-c722b287-10c4-78d0-012e-bbd1685468ba Address:c722b287-10c4-78d0-012e-bbd1685468ba} {Suffrage:Voter ID:server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b Address:2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.706Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:18.706Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.706Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering follower state: follower="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Follower]" leader= - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.706Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering follower state: follower="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Follower]" leader= -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:18.706Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering follower state: follower="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Follower]" leader= - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:18.706Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering follower state: follower="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Follower]" leader= -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=2 - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=2 -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: election won: tally=2 - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: election won: tally=2 -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering leader state: leader="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Leader]" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering leader state: leader="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Leader]" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: added peer, starting replication: peer=server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: added peer, starting replication: peer=server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: pipelining replication: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.776Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: pipelining replication: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.882Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.776609699 +0000 UTC m=+13.106842387 (1 nodes), 69.802838ms from start of poll, 70.050387ms from cluster start. Timeout at 2025-02-09 16:52:18.882144612 +0000 UTC m=+13.212377300, 105.534913ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.882Z [INFO] stable state for Leader reached at 2025-02-09 16:52:18.776609699 +0000 UTC m=+13.106842387 (1 nodes), 69.802838ms from start of poll, 70.050387ms from cluster start. Timeout at 2025-02-09 16:52:18.882144612 +0000 UTC m=+13.212377300, 105.534913ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.887Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:18.887Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:18.993Z [INFO] stable state for Follower reached at 2025-02-09 16:52:18.887818349 +0000 UTC m=+13.218051035 (1 nodes), 709ns from start of poll, 181.259035ms from cluster start. Timeout at 2025-02-09 16:52:18.993686652 +0000 UTC m=+13.323919340, 105.868305ms after stability - testing.go:149: cluster: 2025-02-09T16:52:18.993Z [INFO] stable state for Follower reached at 2025-02-09 16:52:18.887818349 +0000 UTC m=+13.218051035 (1 nodes), 709ns from start of poll, 181.259035ms from cluster start. Timeout at 2025-02-09 16:52:18.993686652 +0000 UTC m=+13.323919340, 105.868305ms after stability - raft_test.go:1492: [INFO] Disconnecting Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Follower] -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.994Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to pipeline appendEntries: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="append pipeline closed" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.994Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to pipeline appendEntries: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="append pipeline closed" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.995Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: aborting pipeline replication: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.995Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: aborting pipeline replication: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.998Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:18.998Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.003Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.003Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.019Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.019Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.020Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.020Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.037Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.037Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to heartbeat to: peer=2e284aaf-108e-94ea-c7e0-08bbc1e65f5b error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to contact: server-id=server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b time=50.804467ms - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to contact: server-id=server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b time=50.804467ms -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to contact quorum of nodes, stepping down - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to contact quorum of nodes, stepping down -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering follower state: follower="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Follower]" leader= - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.039Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering follower state: follower="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Follower]" leader= -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.039Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:19.039Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.040Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.040Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to appendEntries to: peer="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: heartbeat timeout reached, starting election: last-leader=c722b287-10c4-78d0-012e-bbd1685468ba - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: heartbeat timeout reached, starting election: last-leader=c722b287-10c4-78d0-012e-bbd1685468ba -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=3 - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=3 -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.041Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=4 - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=4 -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.092Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=3 - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=3 -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.105Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=5 - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=5 -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.152Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: Election timeout reached, restarting election - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: Election timeout reached, restarting election -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=4 - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=4 -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.196Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.211Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.10593562 +0000 UTC m=+13.436168308 (0 nodes), 66.510475ms from start of poll, 399.376308ms from cluster start. Timeout at 2025-02-09 16:52:19.211887214 +0000 UTC m=+13.542119902, 105.951594ms after stability - testing.go:149: cluster: 2025-02-09T16:52:19.211Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.10593562 +0000 UTC m=+13.436168308 (0 nodes), 66.510475ms from start of poll, 399.376308ms from cluster start. Timeout at 2025-02-09 16:52:19.211887214 +0000 UTC m=+13.542119902, 105.951594ms after stability -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=6 - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=6 -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.215Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: Election timeout reached, restarting election - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [WARN] server-c722b287-10c4-78d0-012e-bbd1685468ba: Election timeout reached, restarting election -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=5 - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [INFO] server-c722b287-10c4-78d0-012e-bbd1685468ba: entering candidate state: node="Node at c722b287-10c4-78d0-012e-bbd1685468ba [Candidate]" term=5 -testLoggerAdapter verbose: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" - testing.go:149: server-c722b287-10c4-78d0-012e-bbd1685468ba: 2025-02-09T16:52:19.277Z [ERROR] server-c722b287-10c4-78d0-012e-bbd1685468ba: failed to make requestVote RPC: target="{Voter server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b}" error="failed to connect to peer: 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b" -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [WARN] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: Election timeout reached, restarting election -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=7 - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [INFO] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: entering candidate state: node="Node at 2e284aaf-108e-94ea-c7e0-08bbc1e65f5b [Candidate]" term=7 -testLoggerAdapter verbose: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" - testing.go:149: server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: 2025-02-09T16:52:19.301Z [ERROR] server-2e284aaf-108e-94ea-c7e0-08bbc1e65f5b: failed to make requestVote RPC: target="{Voter server-c722b287-10c4-78d0-012e-bbd1685468ba c722b287-10c4-78d0-012e-bbd1685468ba}" error="failed to connect to peer: c722b287-10c4-78d0-012e-bbd1685468ba" ---- PASS: TestRaft_LeaderLeaseExpire (0.61s) +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.490Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 Address:e64ee3e1-da7e-b774-e4d8-a37090cc7040} {Suffrage:Voter ID:server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 Address:c8f0a95d-2a4d-e13f-48c8-f79845ea7082}]" + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.490Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 Address:e64ee3e1-da7e-b774-e4d8-a37090cc7040} {Suffrage:Voter ID:server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 Address:c8f0a95d-2a4d-e13f-48c8-f79845ea7082}]" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.490Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 Address:e64ee3e1-da7e-b774-e4d8-a37090cc7040} {Suffrage:Voter ID:server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 Address:c8f0a95d-2a4d-e13f-48c8-f79845ea7082}]" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.490Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 Address:e64ee3e1-da7e-b774-e4d8-a37090cc7040} {Suffrage:Voter ID:server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 Address:c8f0a95d-2a4d-e13f-48c8-f79845ea7082}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.490Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:06.490Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.490Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering follower state: follower="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Follower]" leader= + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.490Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering follower state: follower="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Follower]" leader= +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.491Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering follower state: follower="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Follower]" leader= + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.491Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering follower state: follower="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Follower]" leader= +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.573Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.573Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.573Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=2 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.573Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=2 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=2 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=2 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: duplicate requestVote for same term: term=2 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.574Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.575Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: duplicate requestVote for same term: term=2 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.575Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: duplicate requestVote for same term: term=2 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.633Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.633Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.633Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=3 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.633Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=3 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.634Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering follower state: follower="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Follower]" leader= + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.634Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering follower state: follower="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Follower]" leader= +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: election won: tally=2 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: election won: tally=2 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering leader state: leader="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Leader]" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering leader state: leader="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Leader]" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: added peer, starting replication: peer=server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: added peer, starting replication: peer=server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: pipelining replication: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.634Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: pipelining replication: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.742Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.634361716 +0000 UTC m=+14.786155567 (1 nodes), 143.437404ms from start of poll, 143.778646ms from cluster start. Timeout at 2024-01-08 10:38:06.742013685 +0000 UTC m=+14.893807849, 107.652282ms after stability + testing.go:149: cluster: 2024-01-08T10:38:06.742Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.634361716 +0000 UTC m=+14.786155567 (1 nodes), 143.437404ms from start of poll, 143.778646ms from cluster start. Timeout at 2024-01-08 10:38:06.742013685 +0000 UTC m=+14.893807849, 107.652282ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.754Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:06.754Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.860Z [INFO] stable state for Follower reached at 2024-01-08 10:38:06.754471722 +0000 UTC m=+14.906265828 (1 nodes), 3.408µs from start of poll, 263.888907ms from cluster start. Timeout at 2024-01-08 10:38:06.859986759 +0000 UTC m=+15.011780896, 105.515068ms after stability + testing.go:149: cluster: 2024-01-08T10:38:06.860Z [INFO] stable state for Follower reached at 2024-01-08 10:38:06.754471722 +0000 UTC m=+14.906265828 (1 nodes), 3.408µs from start of poll, 263.888907ms from cluster start. Timeout at 2024-01-08 10:38:06.859986759 +0000 UTC m=+15.011780896, 105.515068ms after stability + raft_test.go:1492: [INFO] Disconnecting Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Follower] +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.865Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to pipeline appendEntries: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="append pipeline closed" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.865Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to pipeline appendEntries: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="append pipeline closed" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.865Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to heartbeat to: peer=e64ee3e1-da7e-b774-e4d8-a37090cc7040 error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.865Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to heartbeat to: peer=e64ee3e1-da7e-b774-e4d8-a37090cc7040 error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.866Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: aborting pipeline replication: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.866Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: aborting pipeline replication: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.882Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to appendEntries to: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.882Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to appendEntries to: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.895Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to heartbeat to: peer=e64ee3e1-da7e-b774-e4d8-a37090cc7040 error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.895Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to heartbeat to: peer=e64ee3e1-da7e-b774-e4d8-a37090cc7040 error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.906Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to appendEntries to: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.906Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to appendEntries to: peer="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.909Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to contact: server-id=server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 time=51.345804ms + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.909Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to contact: server-id=server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 time=51.345804ms +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.910Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to contact quorum of nodes, stepping down + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.910Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to contact quorum of nodes, stepping down +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.910Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering follower state: follower="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Follower]" leader= + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.910Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering follower state: follower="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:06.910Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:06.910Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: heartbeat timeout reached, starting election: last-leader=c8f0a95d-2a4d-e13f-48c8-f79845ea7082 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: heartbeat timeout reached, starting election: last-leader=c8f0a95d-2a4d-e13f-48c8-f79845ea7082 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=4 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=4 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:06.954Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=4 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=4 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:06.994Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=5 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=5 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.022Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.089Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.089Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.089Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=5 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.089Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=5 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.090Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.090Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.101Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.995038847 +0000 UTC m=+15.146833021 (0 nodes), 84.87905ms from start of poll, 504.4561ms from cluster start. Timeout at 2024-01-08 10:38:07.101330799 +0000 UTC m=+15.253124934, 106.291913ms after stability + testing.go:149: cluster: 2024-01-08T10:38:07.101Z [INFO] stable state for Leader reached at 2024-01-08 10:38:06.995038847 +0000 UTC m=+15.146833021 (0 nodes), 84.87905ms from start of poll, 504.4561ms from cluster start. Timeout at 2024-01-08 10:38:07.101330799 +0000 UTC m=+15.253124934, 106.291913ms after stability +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.117Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.117Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.117Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=6 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.117Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=6 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.118Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.118Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.161Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.161Z [WARN] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: Election timeout reached, restarting election +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.162Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=6 + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.162Z [INFO] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: entering candidate state: node="Node at c8f0a95d-2a4d-e13f-48c8-f79845ea7082 [Candidate]" term=6 +testLoggerAdapter verbose: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.162Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" + testing.go:149: server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: 2024-01-08T10:38:07.162Z [ERROR] server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082: failed to make requestVote RPC: target="{Voter server-e64ee3e1-da7e-b774-e4d8-a37090cc7040 e64ee3e1-da7e-b774-e4d8-a37090cc7040}" error="failed to connect to peer: e64ee3e1-da7e-b774-e4d8-a37090cc7040" +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.190Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.190Z [WARN] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: Election timeout reached, restarting election +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.191Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=7 + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.191Z [INFO] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: entering candidate state: node="Node at e64ee3e1-da7e-b774-e4d8-a37090cc7040 [Candidate]" term=7 +testLoggerAdapter verbose: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.191Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" + testing.go:149: server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: 2024-01-08T10:38:07.191Z [ERROR] server-e64ee3e1-da7e-b774-e4d8-a37090cc7040: failed to make requestVote RPC: target="{Voter server-c8f0a95d-2a4d-e13f-48c8-f79845ea7082 c8f0a95d-2a4d-e13f-48c8-f79845ea7082}" error="failed to connect to peer: c8f0a95d-2a4d-e13f-48c8-f79845ea7082" +--- PASS: TestRaft_LeaderLeaseExpire (0.72s) === RUN TestRaft_Barrier -testLoggerAdapter verbose: server-a0d11a06-75af-130f-79f9-fdb8a55698cb: 2025-02-09T16:52:19.317Z [INFO] server-a0d11a06-75af-130f-79f9-fdb8a55698cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" - testing.go:149: server-a0d11a06-75af-130f-79f9-fdb8a55698cb: 2025-02-09T16:52:19.317Z [INFO] server-a0d11a06-75af-130f-79f9-fdb8a55698cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" -testLoggerAdapter verbose: server-60f8640d-a16a-48bc-a470-158993ffe7cc: 2025-02-09T16:52:19.317Z [INFO] server-60f8640d-a16a-48bc-a470-158993ffe7cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" - testing.go:149: server-60f8640d-a16a-48bc-a470-158993ffe7cc: 2025-02-09T16:52:19.317Z [INFO] server-60f8640d-a16a-48bc-a470-158993ffe7cc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.317Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.317Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a0d11a06-75af-130f-79f9-fdb8a55698cb Address:a0d11a06-75af-130f-79f9-fdb8a55698cb} {Suffrage:Voter ID:server-60f8640d-a16a-48bc-a470-158993ffe7cc Address:60f8640d-a16a-48bc-a470-158993ffe7cc} {Suffrage:Voter ID:server-b6323d63-d110-2b16-9070-c017a98118c0 Address:b6323d63-d110-2b16-9070-c017a98118c0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.317Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:19.317Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a0d11a06-75af-130f-79f9-fdb8a55698cb: 2025-02-09T16:52:19.318Z [INFO] server-a0d11a06-75af-130f-79f9-fdb8a55698cb: entering follower state: follower="Node at a0d11a06-75af-130f-79f9-fdb8a55698cb [Follower]" leader= - testing.go:149: server-a0d11a06-75af-130f-79f9-fdb8a55698cb: 2025-02-09T16:52:19.318Z [INFO] server-a0d11a06-75af-130f-79f9-fdb8a55698cb: entering follower state: follower="Node at a0d11a06-75af-130f-79f9-fdb8a55698cb [Follower]" leader= -testLoggerAdapter verbose: server-60f8640d-a16a-48bc-a470-158993ffe7cc: 2025-02-09T16:52:19.318Z [INFO] server-60f8640d-a16a-48bc-a470-158993ffe7cc: entering follower state: follower="Node at 60f8640d-a16a-48bc-a470-158993ffe7cc [Follower]" leader= - testing.go:149: server-60f8640d-a16a-48bc-a470-158993ffe7cc: 2025-02-09T16:52:19.318Z [INFO] server-60f8640d-a16a-48bc-a470-158993ffe7cc: entering follower state: follower="Node at 60f8640d-a16a-48bc-a470-158993ffe7cc [Follower]" leader= -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.318Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering follower state: follower="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Follower]" leader= - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.318Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering follower state: follower="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Follower]" leader= -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [WARN] server-b6323d63-d110-2b16-9070-c017a98118c0: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [WARN] server-b6323d63-d110-2b16-9070-c017a98118c0: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering candidate state: node="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Candidate]" term=2 - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering candidate state: node="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Candidate]" term=2 -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: election won: tally=2 - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: election won: tally=2 -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering leader state: leader="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Leader]" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: entering leader state: leader="Node at b6323d63-d110-2b16-9070-c017a98118c0 [Leader]" -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: added peer, starting replication: peer=server-a0d11a06-75af-130f-79f9-fdb8a55698cb - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: added peer, starting replication: peer=server-a0d11a06-75af-130f-79f9-fdb8a55698cb -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: added peer, starting replication: peer=server-60f8640d-a16a-48bc-a470-158993ffe7cc - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: added peer, starting replication: peer=server-60f8640d-a16a-48bc-a470-158993ffe7cc -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: pipelining replication: peer="{Voter server-60f8640d-a16a-48bc-a470-158993ffe7cc 60f8640d-a16a-48bc-a470-158993ffe7cc}" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: pipelining replication: peer="{Voter server-60f8640d-a16a-48bc-a470-158993ffe7cc 60f8640d-a16a-48bc-a470-158993ffe7cc}" -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: pipelining replication: peer="{Voter server-a0d11a06-75af-130f-79f9-fdb8a55698cb a0d11a06-75af-130f-79f9-fdb8a55698cb}" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.381Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: pipelining replication: peer="{Voter server-a0d11a06-75af-130f-79f9-fdb8a55698cb a0d11a06-75af-130f-79f9-fdb8a55698cb}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.486Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.381498821 +0000 UTC m=+13.711731509 (1 nodes), 63.481209ms from start of poll, 63.905096ms from cluster start. Timeout at 2025-02-09 16:52:19.486552596 +0000 UTC m=+13.816785282, 105.053773ms after stability - testing.go:149: cluster: 2025-02-09T16:52:19.486Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.381498821 +0000 UTC m=+13.711731509 (1 nodes), 63.481209ms from start of poll, 63.905096ms from cluster start. Timeout at 2025-02-09 16:52:19.486552596 +0000 UTC m=+13.816785282, 105.053773ms after stability -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.500Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: aborting pipeline replication: peer="{Voter server-60f8640d-a16a-48bc-a470-158993ffe7cc 60f8640d-a16a-48bc-a470-158993ffe7cc}" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.500Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: aborting pipeline replication: peer="{Voter server-60f8640d-a16a-48bc-a470-158993ffe7cc 60f8640d-a16a-48bc-a470-158993ffe7cc}" -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.500Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: aborting pipeline replication: peer="{Voter server-a0d11a06-75af-130f-79f9-fdb8a55698cb a0d11a06-75af-130f-79f9-fdb8a55698cb}" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.500Z [INFO] server-b6323d63-d110-2b16-9070-c017a98118c0: aborting pipeline replication: peer="{Voter server-a0d11a06-75af-130f-79f9-fdb8a55698cb a0d11a06-75af-130f-79f9-fdb8a55698cb}" -testLoggerAdapter verbose: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.551Z [ERROR] server-b6323d63-d110-2b16-9070-c017a98118c0: failed to heartbeat to: peer=60f8640d-a16a-48bc-a470-158993ffe7cc error="command timed out" - testing.go:149: server-b6323d63-d110-2b16-9070-c017a98118c0: 2025-02-09T16:52:19.551Z [ERROR] server-b6323d63-d110-2b16-9070-c017a98118c0: failed to heartbeat to: peer=60f8640d-a16a-48bc-a470-158993ffe7cc error="command timed out" ---- PASS: TestRaft_Barrier (0.23s) +testLoggerAdapter verbose: server-a909e40e-0ad7-4866-c074-93b668e02bc7: 2024-01-08T10:38:07.212Z [INFO] server-a909e40e-0ad7-4866-c074-93b668e02bc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" + testing.go:149: server-a909e40e-0ad7-4866-c074-93b668e02bc7: 2024-01-08T10:38:07.212Z [INFO] server-a909e40e-0ad7-4866-c074-93b668e02bc7: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" +testLoggerAdapter verbose: server-a909e40e-0ad7-4866-c074-93b668e02bc7: 2024-01-08T10:38:07.213Z [INFO] server-a909e40e-0ad7-4866-c074-93b668e02bc7: entering follower state: follower="Node at a909e40e-0ad7-4866-c074-93b668e02bc7 [Follower]" leader= + testing.go:149: server-a909e40e-0ad7-4866-c074-93b668e02bc7: 2024-01-08T10:38:07.213Z [INFO] server-a909e40e-0ad7-4866-c074-93b668e02bc7: entering follower state: follower="Node at a909e40e-0ad7-4866-c074-93b668e02bc7 [Follower]" leader= +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.214Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.214Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" +testLoggerAdapter verbose: server-c89db727-b52f-cb03-f79b-a3cf61045b7e: 2024-01-08T10:38:07.214Z [INFO] server-c89db727-b52f-cb03-f79b-a3cf61045b7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" + testing.go:149: server-c89db727-b52f-cb03-f79b-a3cf61045b7e: 2024-01-08T10:38:07.214Z [INFO] server-c89db727-b52f-cb03-f79b-a3cf61045b7e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a909e40e-0ad7-4866-c074-93b668e02bc7 Address:a909e40e-0ad7-4866-c074-93b668e02bc7} {Suffrage:Voter ID:server-dd763fb6-a83d-f468-9c58-020ecef0822a Address:dd763fb6-a83d-f468-9c58-020ecef0822a} {Suffrage:Voter ID:server-c89db727-b52f-cb03-f79b-a3cf61045b7e Address:c89db727-b52f-cb03-f79b-a3cf61045b7e}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.215Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:07.215Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.215Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering follower state: follower="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Follower]" leader= + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.215Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering follower state: follower="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Follower]" leader= +testLoggerAdapter verbose: server-c89db727-b52f-cb03-f79b-a3cf61045b7e: 2024-01-08T10:38:07.215Z [INFO] server-c89db727-b52f-cb03-f79b-a3cf61045b7e: entering follower state: follower="Node at c89db727-b52f-cb03-f79b-a3cf61045b7e [Follower]" leader= + testing.go:149: server-c89db727-b52f-cb03-f79b-a3cf61045b7e: 2024-01-08T10:38:07.215Z [INFO] server-c89db727-b52f-cb03-f79b-a3cf61045b7e: entering follower state: follower="Node at c89db727-b52f-cb03-f79b-a3cf61045b7e [Follower]" leader= +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.282Z [WARN] server-dd763fb6-a83d-f468-9c58-020ecef0822a: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.282Z [WARN] server-dd763fb6-a83d-f468-9c58-020ecef0822a: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.282Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering candidate state: node="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Candidate]" term=2 + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.282Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering candidate state: node="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Candidate]" term=2 +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: election won: tally=2 + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: election won: tally=2 +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering leader state: leader="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Leader]" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: entering leader state: leader="Node at dd763fb6-a83d-f468-9c58-020ecef0822a [Leader]" +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: added peer, starting replication: peer=server-a909e40e-0ad7-4866-c074-93b668e02bc7 + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: added peer, starting replication: peer=server-a909e40e-0ad7-4866-c074-93b668e02bc7 +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: added peer, starting replication: peer=server-c89db727-b52f-cb03-f79b-a3cf61045b7e + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.283Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: added peer, starting replication: peer=server-c89db727-b52f-cb03-f79b-a3cf61045b7e +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.284Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: pipelining replication: peer="{Voter server-c89db727-b52f-cb03-f79b-a3cf61045b7e c89db727-b52f-cb03-f79b-a3cf61045b7e}" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.284Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: pipelining replication: peer="{Voter server-c89db727-b52f-cb03-f79b-a3cf61045b7e c89db727-b52f-cb03-f79b-a3cf61045b7e}" +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.285Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: pipelining replication: peer="{Voter server-a909e40e-0ad7-4866-c074-93b668e02bc7 a909e40e-0ad7-4866-c074-93b668e02bc7}" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.285Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: pipelining replication: peer="{Voter server-a909e40e-0ad7-4866-c074-93b668e02bc7 a909e40e-0ad7-4866-c074-93b668e02bc7}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.390Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.284767458 +0000 UTC m=+15.436561592 (1 nodes), 69.444613ms from start of poll, 72.771043ms from cluster start. Timeout at 2024-01-08 10:38:07.390144979 +0000 UTC m=+15.541939092, 105.3775ms after stability + testing.go:149: cluster: 2024-01-08T10:38:07.390Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.284767458 +0000 UTC m=+15.436561592 (1 nodes), 69.444613ms from start of poll, 72.771043ms from cluster start. Timeout at 2024-01-08 10:38:07.390144979 +0000 UTC m=+15.541939092, 105.3775ms after stability +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.422Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: aborting pipeline replication: peer="{Voter server-c89db727-b52f-cb03-f79b-a3cf61045b7e c89db727-b52f-cb03-f79b-a3cf61045b7e}" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.422Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: aborting pipeline replication: peer="{Voter server-c89db727-b52f-cb03-f79b-a3cf61045b7e c89db727-b52f-cb03-f79b-a3cf61045b7e}" +testLoggerAdapter verbose: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.423Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: aborting pipeline replication: peer="{Voter server-a909e40e-0ad7-4866-c074-93b668e02bc7 a909e40e-0ad7-4866-c074-93b668e02bc7}" + testing.go:149: server-dd763fb6-a83d-f468-9c58-020ecef0822a: 2024-01-08T10:38:07.423Z [INFO] server-dd763fb6-a83d-f468-9c58-020ecef0822a: aborting pipeline replication: peer="{Voter server-a909e40e-0ad7-4866-c074-93b668e02bc7 a909e40e-0ad7-4866-c074-93b668e02bc7}" +--- PASS: TestRaft_Barrier (0.21s) === RUN TestRaft_VerifyLeader -testLoggerAdapter verbose: server-08b28ba4-8503-a431-9a28-87dfa6f32568: 2025-02-09T16:52:19.551Z [INFO] server-08b28ba4-8503-a431-9a28-87dfa6f32568: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" - testing.go:149: server-08b28ba4-8503-a431-9a28-87dfa6f32568: 2025-02-09T16:52:19.551Z [INFO] server-08b28ba4-8503-a431-9a28-87dfa6f32568: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" -testLoggerAdapter verbose: server-85464139-332e-d54f-7554-0233c2e9a21d: 2025-02-09T16:52:19.552Z [INFO] server-85464139-332e-d54f-7554-0233c2e9a21d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" - testing.go:149: server-85464139-332e-d54f-7554-0233c2e9a21d: 2025-02-09T16:52:19.552Z [INFO] server-85464139-332e-d54f-7554-0233c2e9a21d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.552Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.552Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-08b28ba4-8503-a431-9a28-87dfa6f32568 Address:08b28ba4-8503-a431-9a28-87dfa6f32568} {Suffrage:Voter ID:server-85464139-332e-d54f-7554-0233c2e9a21d Address:85464139-332e-d54f-7554-0233c2e9a21d} {Suffrage:Voter ID:server-44912792-8f2b-4d12-c2c2-348847e75f63 Address:44912792-8f2b-4d12-c2c2-348847e75f63}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.552Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:19.552Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-08b28ba4-8503-a431-9a28-87dfa6f32568: 2025-02-09T16:52:19.552Z [INFO] server-08b28ba4-8503-a431-9a28-87dfa6f32568: entering follower state: follower="Node at 08b28ba4-8503-a431-9a28-87dfa6f32568 [Follower]" leader= - testing.go:149: server-08b28ba4-8503-a431-9a28-87dfa6f32568: 2025-02-09T16:52:19.552Z [INFO] server-08b28ba4-8503-a431-9a28-87dfa6f32568: entering follower state: follower="Node at 08b28ba4-8503-a431-9a28-87dfa6f32568 [Follower]" leader= -testLoggerAdapter verbose: server-85464139-332e-d54f-7554-0233c2e9a21d: 2025-02-09T16:52:19.552Z [INFO] server-85464139-332e-d54f-7554-0233c2e9a21d: entering follower state: follower="Node at 85464139-332e-d54f-7554-0233c2e9a21d [Follower]" leader= - testing.go:149: server-85464139-332e-d54f-7554-0233c2e9a21d: 2025-02-09T16:52:19.552Z [INFO] server-85464139-332e-d54f-7554-0233c2e9a21d: entering follower state: follower="Node at 85464139-332e-d54f-7554-0233c2e9a21d [Follower]" leader= -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.552Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering follower state: follower="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Follower]" leader= - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.552Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering follower state: follower="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Follower]" leader= -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [WARN] server-44912792-8f2b-4d12-c2c2-348847e75f63: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [WARN] server-44912792-8f2b-4d12-c2c2-348847e75f63: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering candidate state: node="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Candidate]" term=2 - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering candidate state: node="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Candidate]" term=2 -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: election won: tally=2 - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: election won: tally=2 -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering leader state: leader="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Leader]" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: entering leader state: leader="Node at 44912792-8f2b-4d12-c2c2-348847e75f63 [Leader]" -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: added peer, starting replication: peer=server-08b28ba4-8503-a431-9a28-87dfa6f32568 - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: added peer, starting replication: peer=server-08b28ba4-8503-a431-9a28-87dfa6f32568 -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: added peer, starting replication: peer=server-85464139-332e-d54f-7554-0233c2e9a21d - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: added peer, starting replication: peer=server-85464139-332e-d54f-7554-0233c2e9a21d -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: pipelining replication: peer="{Voter server-85464139-332e-d54f-7554-0233c2e9a21d 85464139-332e-d54f-7554-0233c2e9a21d}" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: pipelining replication: peer="{Voter server-85464139-332e-d54f-7554-0233c2e9a21d 85464139-332e-d54f-7554-0233c2e9a21d}" -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: pipelining replication: peer="{Voter server-08b28ba4-8503-a431-9a28-87dfa6f32568 08b28ba4-8503-a431-9a28-87dfa6f32568}" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.607Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: pipelining replication: peer="{Voter server-08b28ba4-8503-a431-9a28-87dfa6f32568 08b28ba4-8503-a431-9a28-87dfa6f32568}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.713Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.607797765 +0000 UTC m=+13.938030453 (1 nodes), 55.60314ms from start of poll, 55.963802ms from cluster start. Timeout at 2025-02-09 16:52:19.712979982 +0000 UTC m=+14.043212670, 105.182217ms after stability - testing.go:149: cluster: 2025-02-09T16:52:19.713Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.607797765 +0000 UTC m=+13.938030453 (1 nodes), 55.60314ms from start of poll, 55.963802ms from cluster start. Timeout at 2025-02-09 16:52:19.712979982 +0000 UTC m=+14.043212670, 105.182217ms after stability -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.713Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: aborting pipeline replication: peer="{Voter server-85464139-332e-d54f-7554-0233c2e9a21d 85464139-332e-d54f-7554-0233c2e9a21d}" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.713Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: aborting pipeline replication: peer="{Voter server-85464139-332e-d54f-7554-0233c2e9a21d 85464139-332e-d54f-7554-0233c2e9a21d}" -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.713Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: aborting pipeline replication: peer="{Voter server-08b28ba4-8503-a431-9a28-87dfa6f32568 08b28ba4-8503-a431-9a28-87dfa6f32568}" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.713Z [INFO] server-44912792-8f2b-4d12-c2c2-348847e75f63: aborting pipeline replication: peer="{Voter server-08b28ba4-8503-a431-9a28-87dfa6f32568 08b28ba4-8503-a431-9a28-87dfa6f32568}" -testLoggerAdapter verbose: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.763Z [ERROR] server-44912792-8f2b-4d12-c2c2-348847e75f63: failed to heartbeat to: peer=08b28ba4-8503-a431-9a28-87dfa6f32568 error="command timed out" - testing.go:149: server-44912792-8f2b-4d12-c2c2-348847e75f63: 2025-02-09T16:52:19.763Z [ERROR] server-44912792-8f2b-4d12-c2c2-348847e75f63: failed to heartbeat to: peer=08b28ba4-8503-a431-9a28-87dfa6f32568 error="command timed out" ---- PASS: TestRaft_VerifyLeader (0.21s) +testLoggerAdapter verbose: server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: 2024-01-08T10:38:07.426Z [INFO] server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" + testing.go:149: server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: 2024-01-08T10:38:07.426Z [INFO] server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.427Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" +testLoggerAdapter verbose: server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: 2024-01-08T10:38:07.427Z [INFO] server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: entering follower state: follower="Node at 9a06e9f4-6df1-37c6-af33-ae7eae76ca73 [Follower]" leader= + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.427Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.428Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering follower state: follower="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Follower]" leader= + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.428Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering follower state: follower="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Follower]" leader= + testing.go:149: server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: 2024-01-08T10:38:07.427Z [INFO] server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73: entering follower state: follower="Node at 9a06e9f4-6df1-37c6-af33-ae7eae76ca73 [Follower]" leader= +testLoggerAdapter verbose: server-7fe15e95-47aa-4c49-bff3-8188cc68453c: 2024-01-08T10:38:07.428Z [INFO] server-7fe15e95-47aa-4c49-bff3-8188cc68453c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" + testing.go:149: server-7fe15e95-47aa-4c49-bff3-8188cc68453c: 2024-01-08T10:38:07.428Z [INFO] server-7fe15e95-47aa-4c49-bff3-8188cc68453c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 Address:9a06e9f4-6df1-37c6-af33-ae7eae76ca73} {Suffrage:Voter ID:server-d2db0cf3-4585-1671-3512-bb0d14f751aa Address:d2db0cf3-4585-1671-3512-bb0d14f751aa} {Suffrage:Voter ID:server-7fe15e95-47aa-4c49-bff3-8188cc68453c Address:7fe15e95-47aa-4c49-bff3-8188cc68453c}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.429Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:07.429Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-7fe15e95-47aa-4c49-bff3-8188cc68453c: 2024-01-08T10:38:07.429Z [INFO] server-7fe15e95-47aa-4c49-bff3-8188cc68453c: entering follower state: follower="Node at 7fe15e95-47aa-4c49-bff3-8188cc68453c [Follower]" leader= + testing.go:149: server-7fe15e95-47aa-4c49-bff3-8188cc68453c: 2024-01-08T10:38:07.429Z [INFO] server-7fe15e95-47aa-4c49-bff3-8188cc68453c: entering follower state: follower="Node at 7fe15e95-47aa-4c49-bff3-8188cc68453c [Follower]" leader= +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.481Z [WARN] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.481Z [WARN] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.481Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering candidate state: node="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Candidate]" term=2 + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.481Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering candidate state: node="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Candidate]" term=2 +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: election won: tally=2 + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: election won: tally=2 +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering leader state: leader="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Leader]" + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: entering leader state: leader="Node at d2db0cf3-4585-1671-3512-bb0d14f751aa [Leader]" +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: added peer, starting replication: peer=server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: added peer, starting replication: peer=server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: added peer, starting replication: peer=server-7fe15e95-47aa-4c49-bff3-8188cc68453c + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.482Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: added peer, starting replication: peer=server-7fe15e95-47aa-4c49-bff3-8188cc68453c +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.483Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: pipelining replication: peer="{Voter server-7fe15e95-47aa-4c49-bff3-8188cc68453c 7fe15e95-47aa-4c49-bff3-8188cc68453c}" + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.483Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: pipelining replication: peer="{Voter server-7fe15e95-47aa-4c49-bff3-8188cc68453c 7fe15e95-47aa-4c49-bff3-8188cc68453c}" +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.483Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: pipelining replication: peer="{Voter server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 9a06e9f4-6df1-37c6-af33-ae7eae76ca73}" + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.483Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: pipelining replication: peer="{Voter server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 9a06e9f4-6df1-37c6-af33-ae7eae76ca73}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.588Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.482482075 +0000 UTC m=+15.634276250 (1 nodes), 53.258956ms from start of poll, 56.071562ms from cluster start. Timeout at 2024-01-08 10:38:07.588140557 +0000 UTC m=+15.739934694, 105.658444ms after stability + testing.go:149: cluster: 2024-01-08T10:38:07.588Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.482482075 +0000 UTC m=+15.634276250 (1 nodes), 53.258956ms from start of poll, 56.071562ms from cluster start. Timeout at 2024-01-08 10:38:07.588140557 +0000 UTC m=+15.739934694, 105.658444ms after stability +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.592Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: aborting pipeline replication: peer="{Voter server-7fe15e95-47aa-4c49-bff3-8188cc68453c 7fe15e95-47aa-4c49-bff3-8188cc68453c}" + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.592Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: aborting pipeline replication: peer="{Voter server-7fe15e95-47aa-4c49-bff3-8188cc68453c 7fe15e95-47aa-4c49-bff3-8188cc68453c}" +testLoggerAdapter verbose: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.592Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: aborting pipeline replication: peer="{Voter server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 9a06e9f4-6df1-37c6-af33-ae7eae76ca73}" + testing.go:149: server-d2db0cf3-4585-1671-3512-bb0d14f751aa: 2024-01-08T10:38:07.592Z [INFO] server-d2db0cf3-4585-1671-3512-bb0d14f751aa: aborting pipeline replication: peer="{Voter server-9a06e9f4-6df1-37c6-af33-ae7eae76ca73 9a06e9f4-6df1-37c6-af33-ae7eae76ca73}" +--- PASS: TestRaft_VerifyLeader (0.17s) === RUN TestRaft_VerifyLeader_Single -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.763Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc Address:403cd8cf-8b68-bea7-3980-bae8bfa40dbc}]" - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.763Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc Address:403cd8cf-8b68-bea7-3980-bae8bfa40dbc}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.763Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:19.763Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.763Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering follower state: follower="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Follower]" leader= - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.763Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering follower state: follower="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Follower]" leader= -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.822Z [WARN] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.822Z [WARN] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering candidate state: node="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Candidate]" term=2 - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering candidate state: node="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Candidate]" term=2 -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: election won: tally=1 - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: election won: tally=1 -testLoggerAdapter verbose: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering leader state: leader="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Leader]" - testing.go:149: server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: 2025-02-09T16:52:19.823Z [INFO] server-403cd8cf-8b68-bea7-3980-bae8bfa40dbc: entering leader state: leader="Node at 403cd8cf-8b68-bea7-3980-bae8bfa40dbc [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.929Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.823135772 +0000 UTC m=+14.153368460 (1 nodes), 59.150395ms from start of poll, 59.322156ms from cluster start. Timeout at 2025-02-09 16:52:19.929163277 +0000 UTC m=+14.259395966, 106.027506ms after stability - testing.go:149: cluster: 2025-02-09T16:52:19.929Z [INFO] stable state for Leader reached at 2025-02-09 16:52:19.823135772 +0000 UTC m=+14.153368460 (1 nodes), 59.150395ms from start of poll, 59.322156ms from cluster start. Timeout at 2025-02-09 16:52:19.929163277 +0000 UTC m=+14.259395966, 106.027506ms after stability ---- PASS: TestRaft_VerifyLeader_Single (0.17s) +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.599Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 Address:f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54}]" + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.599Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 Address:f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.599Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:07.599Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.600Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering follower state: follower="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Follower]" leader= + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.600Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering follower state: follower="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Follower]" leader= +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.674Z [WARN] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.674Z [WARN] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.674Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering candidate state: node="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Candidate]" term=2 + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.674Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering candidate state: node="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Candidate]" term=2 +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.675Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: election won: tally=1 + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.675Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: election won: tally=1 +testLoggerAdapter verbose: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.675Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering leader state: leader="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Leader]" + testing.go:149: server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: 2024-01-08T10:38:07.675Z [INFO] server-f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54: entering leader state: leader="Node at f3d4c3d2-4a15-4249-ecbc-e0aabf3ccb54 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.781Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.67561028 +0000 UTC m=+15.827404414 (1 nodes), 75.476318ms from start of poll, 76.51451ms from cluster start. Timeout at 2024-01-08 10:38:07.781458445 +0000 UTC m=+15.933252579, 105.848165ms after stability + testing.go:149: cluster: 2024-01-08T10:38:07.781Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.67561028 +0000 UTC m=+15.827404414 (1 nodes), 75.476318ms from start of poll, 76.51451ms from cluster start. Timeout at 2024-01-08 10:38:07.781458445 +0000 UTC m=+15.933252579, 105.848165ms after stability +--- PASS: TestRaft_VerifyLeader_Single (0.18s) === RUN TestRaft_VerifyLeader_Fail -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:19.929Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 Address:a3ec91e7-1b0c-6eb2-c17c-59ac20845b41} {Suffrage:Voter ID:server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 Address:6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}]" - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:19.929Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 Address:a3ec91e7-1b0c-6eb2-c17c-59ac20845b41} {Suffrage:Voter ID:server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 Address:6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}]" -testLoggerAdapter verbose: server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: 2025-02-09T16:52:19.929Z [INFO] server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 Address:a3ec91e7-1b0c-6eb2-c17c-59ac20845b41} {Suffrage:Voter ID:server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 Address:6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}]" - testing.go:149: server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: 2025-02-09T16:52:19.929Z [INFO] server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 Address:a3ec91e7-1b0c-6eb2-c17c-59ac20845b41} {Suffrage:Voter ID:server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 Address:6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:19.929Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:19.929Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:19.929Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering follower state: follower="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Follower]" leader= - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:19.929Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering follower state: follower="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Follower]" leader= -testLoggerAdapter verbose: server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: 2025-02-09T16:52:19.930Z [INFO] server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: entering follower state: follower="Node at 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 [Follower]" leader= - testing.go:149: server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: 2025-02-09T16:52:19.930Z [INFO] server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73: entering follower state: follower="Node at 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 [Follower]" leader= -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [WARN] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [WARN] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering candidate state: node="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Candidate]" term=2 - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering candidate state: node="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Candidate]" term=2 -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: election won: tally=2 - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: election won: tally=2 -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering leader state: leader="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Leader]" - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering leader state: leader="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Leader]" -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: added peer, starting replication: peer=server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: added peer, starting replication: peer=server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: pipelining replication: peer="{Voter server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}" - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.014Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: pipelining replication: peer="{Voter server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.120Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.014413435 +0000 UTC m=+14.344646123 (1 nodes), 84.457826ms from start of poll, 84.775587ms from cluster start. Timeout at 2025-02-09 16:52:20.120082914 +0000 UTC m=+14.450315602, 105.669479ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.120Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.014413435 +0000 UTC m=+14.344646123 (1 nodes), 84.457826ms from start of poll, 84.775587ms from cluster start. Timeout at 2025-02-09 16:52:20.120082914 +0000 UTC m=+14.450315602, 105.669479ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.120Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:20.120Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.225Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.12019592 +0000 UTC m=+14.450428606 (1 nodes), 869ns from start of poll, 190.55807ms from cluster start. Timeout at 2025-02-09 16:52:20.225926359 +0000 UTC m=+14.556159045, 105.730439ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.225Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.12019592 +0000 UTC m=+14.450428606 (1 nodes), 869ns from start of poll, 190.55807ms from cluster start. Timeout at 2025-02-09 16:52:20.225926359 +0000 UTC m=+14.556159045, 105.730439ms after stability -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [WARN] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: new leader elected, stepping down - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [WARN] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: new leader elected, stepping down -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering follower state: follower="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Follower]" leader= - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: entering follower state: follower="Node at a3ec91e7-1b0c-6eb2-c17c-59ac20845b41 [Follower]" leader= -testLoggerAdapter verbose: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: aborting pipeline replication: peer="{Voter server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}" - testing.go:149: server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: 2025-02-09T16:52:20.226Z [INFO] server-a3ec91e7-1b0c-6eb2-c17c-59ac20845b41: aborting pipeline replication: peer="{Voter server-6e8a3be0-29d5-e0e2-6b8c-4898e302fd73 6e8a3be0-29d5-e0e2-6b8c-4898e302fd73}" +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.784Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c69e841-fb9a-d527-764c-ea83335c4704 Address:9c69e841-fb9a-d527-764c-ea83335c4704} {Suffrage:Voter ID:server-b825f943-f310-52c0-7aa4-ae50180a9fee Address:b825f943-f310-52c0-7aa4-ae50180a9fee}]" + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.784Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c69e841-fb9a-d527-764c-ea83335c4704 Address:9c69e841-fb9a-d527-764c-ea83335c4704} {Suffrage:Voter ID:server-b825f943-f310-52c0-7aa4-ae50180a9fee Address:b825f943-f310-52c0-7aa4-ae50180a9fee}]" +testLoggerAdapter verbose: server-b825f943-f310-52c0-7aa4-ae50180a9fee: 2024-01-08T10:38:07.785Z [INFO] server-b825f943-f310-52c0-7aa4-ae50180a9fee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c69e841-fb9a-d527-764c-ea83335c4704 Address:9c69e841-fb9a-d527-764c-ea83335c4704} {Suffrage:Voter ID:server-b825f943-f310-52c0-7aa4-ae50180a9fee Address:b825f943-f310-52c0-7aa4-ae50180a9fee}]" +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.785Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering follower state: follower="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Follower]" leader= + testing.go:149: server-b825f943-f310-52c0-7aa4-ae50180a9fee: 2024-01-08T10:38:07.785Z [INFO] server-b825f943-f310-52c0-7aa4-ae50180a9fee: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-9c69e841-fb9a-d527-764c-ea83335c4704 Address:9c69e841-fb9a-d527-764c-ea83335c4704} {Suffrage:Voter ID:server-b825f943-f310-52c0-7aa4-ae50180a9fee Address:b825f943-f310-52c0-7aa4-ae50180a9fee}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.786Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.785Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering follower state: follower="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Follower]" leader= +testLoggerAdapter verbose: server-b825f943-f310-52c0-7aa4-ae50180a9fee: 2024-01-08T10:38:07.786Z [INFO] server-b825f943-f310-52c0-7aa4-ae50180a9fee: entering follower state: follower="Node at b825f943-f310-52c0-7aa4-ae50180a9fee [Follower]" leader= + testing.go:149: server-b825f943-f310-52c0-7aa4-ae50180a9fee: 2024-01-08T10:38:07.786Z [INFO] server-b825f943-f310-52c0-7aa4-ae50180a9fee: entering follower state: follower="Node at b825f943-f310-52c0-7aa4-ae50180a9fee [Follower]" leader= + testing.go:149: cluster: 2024-01-08T10:38:07.786Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.869Z [WARN] server-9c69e841-fb9a-d527-764c-ea83335c4704: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.869Z [WARN] server-9c69e841-fb9a-d527-764c-ea83335c4704: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.869Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering candidate state: node="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Candidate]" term=2 + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.869Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering candidate state: node="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Candidate]" term=2 +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: election won: tally=2 + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: election won: tally=2 +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering leader state: leader="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Leader]" + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering leader state: leader="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Leader]" +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: added peer, starting replication: peer=server-b825f943-f310-52c0-7aa4-ae50180a9fee + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.870Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: added peer, starting replication: peer=server-b825f943-f310-52c0-7aa4-ae50180a9fee +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.871Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: pipelining replication: peer="{Voter server-b825f943-f310-52c0-7aa4-ae50180a9fee b825f943-f310-52c0-7aa4-ae50180a9fee}" + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:07.871Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: pipelining replication: peer="{Voter server-b825f943-f310-52c0-7aa4-ae50180a9fee b825f943-f310-52c0-7aa4-ae50180a9fee}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.976Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.871354104 +0000 UTC m=+16.023148263 (1 nodes), 84.256402ms from start of poll, 87.073938ms from cluster start. Timeout at 2024-01-08 10:38:07.976631059 +0000 UTC m=+16.128425197, 105.276934ms after stability + testing.go:149: cluster: 2024-01-08T10:38:07.976Z [INFO] stable state for Leader reached at 2024-01-08 10:38:07.871354104 +0000 UTC m=+16.023148263 (1 nodes), 84.256402ms from start of poll, 87.073938ms from cluster start. Timeout at 2024-01-08 10:38:07.976631059 +0000 UTC m=+16.128425197, 105.276934ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:07.977Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:07.977Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.083Z [INFO] stable state for Follower reached at 2024-01-08 10:38:07.977375547 +0000 UTC m=+16.129169671 (1 nodes), 3.079µs from start of poll, 193.095346ms from cluster start. Timeout at 2024-01-08 10:38:08.082610748 +0000 UTC m=+16.234404897, 105.235226ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.083Z [INFO] stable state for Follower reached at 2024-01-08 10:38:07.977375547 +0000 UTC m=+16.129169671 (1 nodes), 3.079µs from start of poll, 193.095346ms from cluster start. Timeout at 2024-01-08 10:38:08.082610748 +0000 UTC m=+16.234404897, 105.235226ms after stability +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [WARN] server-9c69e841-fb9a-d527-764c-ea83335c4704: new leader elected, stepping down + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [WARN] server-9c69e841-fb9a-d527-764c-ea83335c4704: new leader elected, stepping down +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering follower state: follower="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Follower]" leader= + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: entering follower state: follower="Node at 9c69e841-fb9a-d527-764c-ea83335c4704 [Follower]" leader= +testLoggerAdapter verbose: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: aborting pipeline replication: peer="{Voter server-b825f943-f310-52c0-7aa4-ae50180a9fee b825f943-f310-52c0-7aa4-ae50180a9fee}" + testing.go:149: server-9c69e841-fb9a-d527-764c-ea83335c4704: 2024-01-08T10:38:08.083Z [INFO] server-9c69e841-fb9a-d527-764c-ea83335c4704: aborting pipeline replication: peer="{Voter server-b825f943-f310-52c0-7aa4-ae50180a9fee b825f943-f310-52c0-7aa4-ae50180a9fee}" --- PASS: TestRaft_VerifyLeader_Fail (0.30s) === RUN TestRaft_VerifyLeader_PartialConnect -testLoggerAdapter verbose: server-301f5a03-2f96-29bc-5b48-e522fb6764f6: 2025-02-09T16:52:20.226Z [INFO] server-301f5a03-2f96-29bc-5b48-e522fb6764f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" - testing.go:149: server-301f5a03-2f96-29bc-5b48-e522fb6764f6: 2025-02-09T16:52:20.226Z [INFO] server-301f5a03-2f96-29bc-5b48-e522fb6764f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" -testLoggerAdapter verbose: server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: 2025-02-09T16:52:20.226Z [INFO] server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" - testing.go:149: server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: 2025-02-09T16:52:20.226Z [INFO] server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.226Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.226Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-301f5a03-2f96-29bc-5b48-e522fb6764f6 Address:301f5a03-2f96-29bc-5b48-e522fb6764f6} {Suffrage:Voter ID:server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f Address:cac67c1a-90a2-a132-dd3a-85ef2b081f0f} {Suffrage:Voter ID:server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88 Address:42b54c2d-9710-63b8-b1a2-5a3a27e78e88}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.226Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:20.226Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-301f5a03-2f96-29bc-5b48-e522fb6764f6: 2025-02-09T16:52:20.226Z [INFO] server-301f5a03-2f96-29bc-5b48-e522fb6764f6: entering follower state: follower="Node at 301f5a03-2f96-29bc-5b48-e522fb6764f6 [Follower]" leader= - testing.go:149: server-301f5a03-2f96-29bc-5b48-e522fb6764f6: 2025-02-09T16:52:20.226Z [INFO] server-301f5a03-2f96-29bc-5b48-e522fb6764f6: entering follower state: follower="Node at 301f5a03-2f96-29bc-5b48-e522fb6764f6 [Follower]" leader= -testLoggerAdapter verbose: server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: 2025-02-09T16:52:20.227Z [INFO] server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: entering follower state: follower="Node at cac67c1a-90a2-a132-dd3a-85ef2b081f0f [Follower]" leader= - testing.go:149: server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: 2025-02-09T16:52:20.227Z [INFO] server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f: entering follower state: follower="Node at cac67c1a-90a2-a132-dd3a-85ef2b081f0f [Follower]" leader= -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.227Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering follower state: follower="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Follower]" leader= - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.227Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering follower state: follower="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Follower]" leader= -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.307Z [WARN] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.307Z [WARN] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering candidate state: node="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Candidate]" term=2 - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering candidate state: node="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Candidate]" term=2 -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: election won: tally=2 - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: election won: tally=2 -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering leader state: leader="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Leader]" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: entering leader state: leader="Node at 42b54c2d-9710-63b8-b1a2-5a3a27e78e88 [Leader]" -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: added peer, starting replication: peer=server-301f5a03-2f96-29bc-5b48-e522fb6764f6 - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: added peer, starting replication: peer=server-301f5a03-2f96-29bc-5b48-e522fb6764f6 -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: added peer, starting replication: peer=server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: added peer, starting replication: peer=server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: pipelining replication: peer="{Voter server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f cac67c1a-90a2-a132-dd3a-85ef2b081f0f}" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: pipelining replication: peer="{Voter server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f cac67c1a-90a2-a132-dd3a-85ef2b081f0f}" -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: pipelining replication: peer="{Voter server-301f5a03-2f96-29bc-5b48-e522fb6764f6 301f5a03-2f96-29bc-5b48-e522fb6764f6}" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.308Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: pipelining replication: peer="{Voter server-301f5a03-2f96-29bc-5b48-e522fb6764f6 301f5a03-2f96-29bc-5b48-e522fb6764f6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.414Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.308433 +0000 UTC m=+14.638665688 (1 nodes), 81.450556ms from start of poll, 81.802512ms from cluster start. Timeout at 2025-02-09 16:52:20.41425244 +0000 UTC m=+14.744485128, 105.81944ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.414Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.308433 +0000 UTC m=+14.638665688 (1 nodes), 81.450556ms from start of poll, 81.802512ms from cluster start. Timeout at 2025-02-09 16:52:20.41425244 +0000 UTC m=+14.744485128, 105.81944ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.420Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:20.420Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.525Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.420077359 +0000 UTC m=+14.750310045 (2 nodes), 1.731µs from start of poll, 193.446869ms from cluster start. Timeout at 2025-02-09 16:52:20.52576234 +0000 UTC m=+14.855995028, 105.684983ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.525Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.420077359 +0000 UTC m=+14.750310045 (2 nodes), 1.731µs from start of poll, 193.446869ms from cluster start. Timeout at 2025-02-09 16:52:20.52576234 +0000 UTC m=+14.855995028, 105.684983ms after stability - raft_test.go:1647: [INFO] Disconnecting Node at 301f5a03-2f96-29bc-5b48-e522fb6764f6 [Follower] -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.525Z [ERROR] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: failed to heartbeat to: peer=301f5a03-2f96-29bc-5b48-e522fb6764f6 error="failed to connect to peer: 301f5a03-2f96-29bc-5b48-e522fb6764f6" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.525Z [ERROR] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: failed to heartbeat to: peer=301f5a03-2f96-29bc-5b48-e522fb6764f6 error="failed to connect to peer: 301f5a03-2f96-29bc-5b48-e522fb6764f6" -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.525Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: aborting pipeline replication: peer="{Voter server-301f5a03-2f96-29bc-5b48-e522fb6764f6 301f5a03-2f96-29bc-5b48-e522fb6764f6}" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.525Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: aborting pipeline replication: peer="{Voter server-301f5a03-2f96-29bc-5b48-e522fb6764f6 301f5a03-2f96-29bc-5b48-e522fb6764f6}" -testLoggerAdapter verbose: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.526Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: aborting pipeline replication: peer="{Voter server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f cac67c1a-90a2-a132-dd3a-85ef2b081f0f}" - testing.go:149: server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: 2025-02-09T16:52:20.526Z [INFO] server-42b54c2d-9710-63b8-b1a2-5a3a27e78e88: aborting pipeline replication: peer="{Voter server-cac67c1a-90a2-a132-dd3a-85ef2b081f0f cac67c1a-90a2-a132-dd3a-85ef2b081f0f}" ---- PASS: TestRaft_VerifyLeader_PartialConnect (0.30s) +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.087Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.087Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" +testLoggerAdapter verbose: server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: 2024-01-08T10:38:08.088Z [INFO] server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" + testing.go:149: server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: 2024-01-08T10:38:08.088Z [INFO] server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" +testLoggerAdapter verbose: server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: 2024-01-08T10:38:08.089Z [INFO] server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" + testing.go:149: server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: 2024-01-08T10:38:08.089Z [INFO] server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-fc7e76de-322a-da98-37be-8643c949617b Address:fc7e76de-322a-da98-37be-8643c949617b} {Suffrage:Voter ID:server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 Address:dcab4873-8ebb-f008-f54c-f6bf2b6d6a48} {Suffrage:Voter ID:server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 Address:576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.089Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:08.089Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.089Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering follower state: follower="Node at fc7e76de-322a-da98-37be-8643c949617b [Follower]" leader= + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.089Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering follower state: follower="Node at fc7e76de-322a-da98-37be-8643c949617b [Follower]" leader= +testLoggerAdapter verbose: server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: 2024-01-08T10:38:08.090Z [INFO] server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: entering follower state: follower="Node at 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 [Follower]" leader= + testing.go:149: server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: 2024-01-08T10:38:08.090Z [INFO] server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2: entering follower state: follower="Node at 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 [Follower]" leader= +testLoggerAdapter verbose: server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: 2024-01-08T10:38:08.091Z [INFO] server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: entering follower state: follower="Node at dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 [Follower]" leader= + testing.go:149: server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: 2024-01-08T10:38:08.091Z [INFO] server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48: entering follower state: follower="Node at dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 [Follower]" leader= +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.152Z [WARN] server-fc7e76de-322a-da98-37be-8643c949617b: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.152Z [WARN] server-fc7e76de-322a-da98-37be-8643c949617b: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering candidate state: node="Node at fc7e76de-322a-da98-37be-8643c949617b [Candidate]" term=2 + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering candidate state: node="Node at fc7e76de-322a-da98-37be-8643c949617b [Candidate]" term=2 +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: election won: tally=2 + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: election won: tally=2 +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering leader state: leader="Node at fc7e76de-322a-da98-37be-8643c949617b [Leader]" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: entering leader state: leader="Node at fc7e76de-322a-da98-37be-8643c949617b [Leader]" +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: added peer, starting replication: peer=server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.153Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: added peer, starting replication: peer=server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.154Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: added peer, starting replication: peer=server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.154Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: added peer, starting replication: peer=server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.154Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: pipelining replication: peer="{Voter server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.154Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: pipelining replication: peer="{Voter server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}" +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.155Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: pipelining replication: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.155Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: pipelining replication: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.262Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.155626558 +0000 UTC m=+16.307420696 (1 nodes), 66.243774ms from start of poll, 68.22877ms from cluster start. Timeout at 2024-01-08 10:38:08.262253569 +0000 UTC m=+16.414047693, 106.626997ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.262Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.155626558 +0000 UTC m=+16.307420696 (1 nodes), 66.243774ms from start of poll, 68.22877ms from cluster start. Timeout at 2024-01-08 10:38:08.262253569 +0000 UTC m=+16.414047693, 106.626997ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.268Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:08.268Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.375Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.268699729 +0000 UTC m=+16.420493837 (2 nodes), 4.705µs from start of poll, 181.301911ms from cluster start. Timeout at 2024-01-08 10:38:08.37553774 +0000 UTC m=+16.527331862, 106.838025ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.375Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.268699729 +0000 UTC m=+16.420493837 (2 nodes), 4.705µs from start of poll, 181.301911ms from cluster start. Timeout at 2024-01-08 10:38:08.37553774 +0000 UTC m=+16.527331862, 106.838025ms after stability + raft_test.go:1647: [INFO] Disconnecting Node at dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 [Follower] +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.376Z [ERROR] server-fc7e76de-322a-da98-37be-8643c949617b: failed to heartbeat to: peer=dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 error="failed to connect to peer: dcab4873-8ebb-f008-f54c-f6bf2b6d6a48" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.376Z [ERROR] server-fc7e76de-322a-da98-37be-8643c949617b: failed to heartbeat to: peer=dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 error="failed to connect to peer: dcab4873-8ebb-f008-f54c-f6bf2b6d6a48" +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [ERROR] server-fc7e76de-322a-da98-37be-8643c949617b: failed to pipeline appendEntries: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" error="append pipeline closed" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [ERROR] server-fc7e76de-322a-da98-37be-8643c949617b: failed to pipeline appendEntries: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" error="append pipeline closed" +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: aborting pipeline replication: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: aborting pipeline replication: peer="{Voter server-dcab4873-8ebb-f008-f54c-f6bf2b6d6a48 dcab4873-8ebb-f008-f54c-f6bf2b6d6a48}" +testLoggerAdapter verbose: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: aborting pipeline replication: peer="{Voter server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}" + testing.go:149: server-fc7e76de-322a-da98-37be-8643c949617b: 2024-01-08T10:38:08.377Z [INFO] server-fc7e76de-322a-da98-37be-8643c949617b: aborting pipeline replication: peer="{Voter server-576e3b92-8e15-2a9a-62a1-40b5ab4b5da2 576e3b92-8e15-2a9a-62a1-40b5ab4b5da2}" +--- PASS: TestRaft_VerifyLeader_PartialConnect (0.29s) === RUN TestRaft_StartAsLeader -testLoggerAdapter verbose: server-2eca0b9c-010a-eaa8-a681-90df706295db: 2025-02-09T16:52:20.526Z [INFO] server-2eca0b9c-010a-eaa8-a681-90df706295db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2eca0b9c-010a-eaa8-a681-90df706295db Address:2eca0b9c-010a-eaa8-a681-90df706295db}]" - testing.go:149: server-2eca0b9c-010a-eaa8-a681-90df706295db: 2025-02-09T16:52:20.526Z [INFO] server-2eca0b9c-010a-eaa8-a681-90df706295db: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2eca0b9c-010a-eaa8-a681-90df706295db Address:2eca0b9c-010a-eaa8-a681-90df706295db}]" -testLoggerAdapter verbose: server-2eca0b9c-010a-eaa8-a681-90df706295db: 2025-02-09T16:52:20.526Z [INFO] server-2eca0b9c-010a-eaa8-a681-90df706295db: entering leader state: leader="Node at 2eca0b9c-010a-eaa8-a681-90df706295db [Leader]" - testing.go:149: server-2eca0b9c-010a-eaa8-a681-90df706295db: 2025-02-09T16:52:20.526Z [INFO] server-2eca0b9c-010a-eaa8-a681-90df706295db: entering leader state: leader="Node at 2eca0b9c-010a-eaa8-a681-90df706295db [Leader]" +testLoggerAdapter verbose: server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: 2024-01-08T10:38:08.381Z [INFO] server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06783dd9-7d4f-27a4-4741-f1635ea4f74f Address:06783dd9-7d4f-27a4-4741-f1635ea4f74f}]" + testing.go:149: server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: 2024-01-08T10:38:08.381Z [INFO] server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-06783dd9-7d4f-27a4-4741-f1635ea4f74f Address:06783dd9-7d4f-27a4-4741-f1635ea4f74f}]" +testLoggerAdapter verbose: server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: 2024-01-08T10:38:08.381Z [INFO] server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: entering leader state: leader="Node at 06783dd9-7d4f-27a4-4741-f1635ea4f74f [Leader]" + testing.go:149: server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: 2024-01-08T10:38:08.381Z [INFO] server-06783dd9-7d4f-27a4-4741-f1635ea4f74f: entering leader state: leader="Node at 06783dd9-7d4f-27a4-4741-f1635ea4f74f [Leader]" --- PASS: TestRaft_StartAsLeader (0.00s) === RUN TestRaft_NotifyCh -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.526Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e62b659d-f5c3-975c-88ab-3505dabab6e8 Address:e62b659d-f5c3-975c-88ab-3505dabab6e8}]" - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.526Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e62b659d-f5c3-975c-88ab-3505dabab6e8 Address:e62b659d-f5c3-975c-88ab-3505dabab6e8}]" -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.526Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering follower state: follower="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Follower]" leader= - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.526Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering follower state: follower="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Follower]" leader= -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [WARN] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [WARN] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering candidate state: node="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Candidate]" term=2 - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering candidate state: node="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Candidate]" term=2 -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: election won: tally=1 - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: election won: tally=1 -testLoggerAdapter verbose: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering leader state: leader="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Leader]" - testing.go:149: server-e62b659d-f5c3-975c-88ab-3505dabab6e8: 2025-02-09T16:52:20.612Z [INFO] server-e62b659d-f5c3-975c-88ab-3505dabab6e8: entering leader state: leader="Node at e62b659d-f5c3-975c-88ab-3505dabab6e8 [Leader]" +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.384Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-589315db-85e9-7634-7828-e72f469725f6 Address:589315db-85e9-7634-7828-e72f469725f6}]" + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.384Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-589315db-85e9-7634-7828-e72f469725f6 Address:589315db-85e9-7634-7828-e72f469725f6}]" +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.385Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering follower state: follower="Node at 589315db-85e9-7634-7828-e72f469725f6 [Follower]" leader= + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.385Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering follower state: follower="Node at 589315db-85e9-7634-7828-e72f469725f6 [Follower]" leader= +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [WARN] server-589315db-85e9-7634-7828-e72f469725f6: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [WARN] server-589315db-85e9-7634-7828-e72f469725f6: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering candidate state: node="Node at 589315db-85e9-7634-7828-e72f469725f6 [Candidate]" term=2 + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering candidate state: node="Node at 589315db-85e9-7634-7828-e72f469725f6 [Candidate]" term=2 +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: election won: tally=1 + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: election won: tally=1 +testLoggerAdapter verbose: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering leader state: leader="Node at 589315db-85e9-7634-7828-e72f469725f6 [Leader]" + testing.go:149: server-589315db-85e9-7634-7828-e72f469725f6: 2024-01-08T10:38:08.476Z [INFO] server-589315db-85e9-7634-7828-e72f469725f6: entering leader state: leader="Node at 589315db-85e9-7634-7828-e72f469725f6 [Leader]" --- PASS: TestRaft_NotifyCh (0.09s) === RUN TestRaft_Voting -testLoggerAdapter verbose: server-3be68524-fddc-467f-fdec-f67e71659a69: 2025-02-09T16:52:20.613Z [INFO] server-3be68524-fddc-467f-fdec-f67e71659a69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" - testing.go:149: server-3be68524-fddc-467f-fdec-f67e71659a69: 2025-02-09T16:52:20.613Z [INFO] server-3be68524-fddc-467f-fdec-f67e71659a69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.613Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.613Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" -testLoggerAdapter verbose: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.613Z [INFO] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" - testing.go:149: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.613Z [INFO] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3be68524-fddc-467f-fdec-f67e71659a69 Address:3be68524-fddc-467f-fdec-f67e71659a69} {Suffrage:Voter ID:server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d Address:b96e2dd7-85a2-89fd-d2c1-626d931bb44d} {Suffrage:Voter ID:server-43e1c16f-042f-32cd-94e1-57f16143a7fb Address:43e1c16f-042f-32cd-94e1-57f16143a7fb}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.613Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:20.613Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-3be68524-fddc-467f-fdec-f67e71659a69: 2025-02-09T16:52:20.613Z [INFO] server-3be68524-fddc-467f-fdec-f67e71659a69: entering follower state: follower="Node at 3be68524-fddc-467f-fdec-f67e71659a69 [Follower]" leader= - testing.go:149: server-3be68524-fddc-467f-fdec-f67e71659a69: 2025-02-09T16:52:20.613Z [INFO] server-3be68524-fddc-467f-fdec-f67e71659a69: entering follower state: follower="Node at 3be68524-fddc-467f-fdec-f67e71659a69 [Follower]" leader= -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.613Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering follower state: follower="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Follower]" leader= - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.613Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering follower state: follower="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Follower]" leader= -testLoggerAdapter verbose: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.613Z [INFO] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: entering follower state: follower="Node at 43e1c16f-042f-32cd-94e1-57f16143a7fb [Follower]" leader= - testing.go:149: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.613Z [INFO] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: entering follower state: follower="Node at 43e1c16f-042f-32cd-94e1-57f16143a7fb [Follower]" leader= -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [WARN] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [WARN] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering candidate state: node="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Candidate]" term=2 - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering candidate state: node="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Candidate]" term=2 -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: election won: tally=2 - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: election won: tally=2 -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering leader state: leader="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Leader]" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: entering leader state: leader="Node at b96e2dd7-85a2-89fd-d2c1-626d931bb44d [Leader]" -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: added peer, starting replication: peer=server-3be68524-fddc-467f-fdec-f67e71659a69 - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: added peer, starting replication: peer=server-3be68524-fddc-467f-fdec-f67e71659a69 -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: added peer, starting replication: peer=server-43e1c16f-042f-32cd-94e1-57f16143a7fb - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.680Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: added peer, starting replication: peer=server-43e1c16f-042f-32cd-94e1-57f16143a7fb -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.681Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: pipelining replication: peer="{Voter server-43e1c16f-042f-32cd-94e1-57f16143a7fb 43e1c16f-042f-32cd-94e1-57f16143a7fb}" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.681Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: pipelining replication: peer="{Voter server-43e1c16f-042f-32cd-94e1-57f16143a7fb 43e1c16f-042f-32cd-94e1-57f16143a7fb}" -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.681Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: pipelining replication: peer="{Voter server-3be68524-fddc-467f-fdec-f67e71659a69 3be68524-fddc-467f-fdec-f67e71659a69}" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.681Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: pipelining replication: peer="{Voter server-3be68524-fddc-467f-fdec-f67e71659a69 3be68524-fddc-467f-fdec-f67e71659a69}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.786Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.681219281 +0000 UTC m=+15.011451969 (2 nodes), 67.831459ms from start of poll, 68.194374ms from cluster start. Timeout at 2025-02-09 16:52:20.786405657 +0000 UTC m=+15.116638345, 105.186376ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.786Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.681219281 +0000 UTC m=+15.011451969 (2 nodes), 67.831459ms from start of poll, 68.194374ms from cluster start. Timeout at 2025-02-09 16:52:20.786405657 +0000 UTC m=+15.116638345, 105.186376ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.786Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:20.786Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.891Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.786527505 +0000 UTC m=+15.116760191 (1 nodes), 881ns from start of poll, 173.502596ms from cluster start. Timeout at 2025-02-09 16:52:20.891865807 +0000 UTC m=+15.222098495, 105.338304ms after stability - testing.go:149: cluster: 2025-02-09T16:52:20.891Z [INFO] stable state for Leader reached at 2025-02-09 16:52:20.786527505 +0000 UTC m=+15.116760191 (1 nodes), 881ns from start of poll, 173.502596ms from cluster start. Timeout at 2025-02-09 16:52:20.891865807 +0000 UTC m=+15.222098495, 105.338304ms after stability -testLoggerAdapter verbose: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.891Z [WARN] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: rejecting vote request since we have a leader: from=3be68524-fddc-467f-fdec-f67e71659a69 leader=b96e2dd7-85a2-89fd-d2c1-626d931bb44d - testing.go:149: server-43e1c16f-042f-32cd-94e1-57f16143a7fb: 2025-02-09T16:52:20.891Z [WARN] server-43e1c16f-042f-32cd-94e1-57f16143a7fb: rejecting vote request since we have a leader: from=3be68524-fddc-467f-fdec-f67e71659a69 leader=b96e2dd7-85a2-89fd-d2c1-626d931bb44d -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.892Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: aborting pipeline replication: peer="{Voter server-43e1c16f-042f-32cd-94e1-57f16143a7fb 43e1c16f-042f-32cd-94e1-57f16143a7fb}" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.892Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: aborting pipeline replication: peer="{Voter server-43e1c16f-042f-32cd-94e1-57f16143a7fb 43e1c16f-042f-32cd-94e1-57f16143a7fb}" -testLoggerAdapter verbose: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.892Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: aborting pipeline replication: peer="{Voter server-3be68524-fddc-467f-fdec-f67e71659a69 3be68524-fddc-467f-fdec-f67e71659a69}" - testing.go:149: server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: 2025-02-09T16:52:20.892Z [INFO] server-b96e2dd7-85a2-89fd-d2c1-626d931bb44d: aborting pipeline replication: peer="{Voter server-3be68524-fddc-467f-fdec-f67e71659a69 3be68524-fddc-467f-fdec-f67e71659a69}" ---- PASS: TestRaft_Voting (0.28s) +testLoggerAdapter verbose: server-81277984-cf6e-3452-2f85-a634a8926c27: 2024-01-08T10:38:08.480Z [INFO] server-81277984-cf6e-3452-2f85-a634a8926c27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" + testing.go:149: server-81277984-cf6e-3452-2f85-a634a8926c27: 2024-01-08T10:38:08.480Z [INFO] server-81277984-cf6e-3452-2f85-a634a8926c27: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" +testLoggerAdapter verbose: server-81277984-cf6e-3452-2f85-a634a8926c27: 2024-01-08T10:38:08.481Z [INFO] server-81277984-cf6e-3452-2f85-a634a8926c27: entering follower state: follower="Node at 81277984-cf6e-3452-2f85-a634a8926c27 [Follower]" leader= +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.482Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" + testing.go:149: server-81277984-cf6e-3452-2f85-a634a8926c27: 2024-01-08T10:38:08.481Z [INFO] server-81277984-cf6e-3452-2f85-a634a8926c27: entering follower state: follower="Node at 81277984-cf6e-3452-2f85-a634a8926c27 [Follower]" leader= + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.482Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.482Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering follower state: follower="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Follower]" leader= + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.482Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering follower state: follower="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Follower]" leader= +testLoggerAdapter verbose: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.483Z [INFO] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" + testing.go:149: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.483Z [INFO] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-81277984-cf6e-3452-2f85-a634a8926c27 Address:81277984-cf6e-3452-2f85-a634a8926c27} {Suffrage:Voter ID:server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac Address:eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac} {Suffrage:Voter ID:server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f Address:57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.483Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:08.483Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.483Z [INFO] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: entering follower state: follower="Node at 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f [Follower]" leader= + testing.go:149: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.483Z [INFO] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: entering follower state: follower="Node at 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f [Follower]" leader= +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [WARN] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [WARN] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering candidate state: node="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Candidate]" term=2 + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering candidate state: node="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Candidate]" term=2 +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: election won: tally=2 + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.562Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: election won: tally=2 +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering leader state: leader="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Leader]" + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: entering leader state: leader="Node at eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac [Leader]" +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: added peer, starting replication: peer=server-81277984-cf6e-3452-2f85-a634a8926c27 + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: added peer, starting replication: peer=server-81277984-cf6e-3452-2f85-a634a8926c27 +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: added peer, starting replication: peer=server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: added peer, starting replication: peer=server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: pipelining replication: peer="{Voter server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}" + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.563Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: pipelining replication: peer="{Voter server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}" +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.564Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: pipelining replication: peer="{Voter server-81277984-cf6e-3452-2f85-a634a8926c27 81277984-cf6e-3452-2f85-a634a8926c27}" + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.564Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: pipelining replication: peer="{Voter server-81277984-cf6e-3452-2f85-a634a8926c27 81277984-cf6e-3452-2f85-a634a8926c27}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.670Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.565031232 +0000 UTC m=+16.716825402 (2 nodes), 81.299025ms from start of poll, 84.927466ms from cluster start. Timeout at 2024-01-08 10:38:08.670050389 +0000 UTC m=+16.821844509, 105.019107ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.670Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.565031232 +0000 UTC m=+16.716825402 (2 nodes), 81.299025ms from start of poll, 84.927466ms from cluster start. Timeout at 2024-01-08 10:38:08.670050389 +0000 UTC m=+16.821844509, 105.019107ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.670Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:08.670Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.776Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.670663994 +0000 UTC m=+16.822458106 (1 nodes), 3.267µs from start of poll, 190.56017ms from cluster start. Timeout at 2024-01-08 10:38:08.775892195 +0000 UTC m=+16.927686302, 105.228196ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.776Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.670663994 +0000 UTC m=+16.822458106 (1 nodes), 3.267µs from start of poll, 190.56017ms from cluster start. Timeout at 2024-01-08 10:38:08.775892195 +0000 UTC m=+16.927686302, 105.228196ms after stability +testLoggerAdapter verbose: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.776Z [WARN] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: rejecting vote request since we have a leader: from=81277984-cf6e-3452-2f85-a634a8926c27 leader=eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac + testing.go:149: server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: 2024-01-08T10:38:08.776Z [WARN] server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f: rejecting vote request since we have a leader: from=81277984-cf6e-3452-2f85-a634a8926c27 leader=eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.777Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: aborting pipeline replication: peer="{Voter server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}" + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.777Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: aborting pipeline replication: peer="{Voter server-57b1cab5-0192-b351-54d4-9cd0ac0d4b3f 57b1cab5-0192-b351-54d4-9cd0ac0d4b3f}" +testLoggerAdapter verbose: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.777Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: aborting pipeline replication: peer="{Voter server-81277984-cf6e-3452-2f85-a634a8926c27 81277984-cf6e-3452-2f85-a634a8926c27}" + testing.go:149: server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: 2024-01-08T10:38:08.777Z [INFO] server-eb1ad3c0-6db3-0384-4aa8-19c0d932e9ac: aborting pipeline replication: peer="{Voter server-81277984-cf6e-3452-2f85-a634a8926c27 81277984-cf6e-3452-2f85-a634a8926c27}" +--- PASS: TestRaft_Voting (0.30s) === RUN TestRaft_ProtocolVersion_RejectRPC -testLoggerAdapter verbose: server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: 2025-02-09T16:52:20.892Z [INFO] server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" - testing.go:149: server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: 2025-02-09T16:52:20.892Z [INFO] server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" -testLoggerAdapter verbose: server-74c08026-5219-806f-4c3e-149fcd4e57b9: 2025-02-09T16:52:20.892Z [INFO] server-74c08026-5219-806f-4c3e-149fcd4e57b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" - testing.go:149: server-74c08026-5219-806f-4c3e-149fcd4e57b9: 2025-02-09T16:52:20.892Z [INFO] server-74c08026-5219-806f-4c3e-149fcd4e57b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.892Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.892Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad Address:a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad} {Suffrage:Voter ID:server-74c08026-5219-806f-4c3e-149fcd4e57b9 Address:74c08026-5219-806f-4c3e-149fcd4e57b9} {Suffrage:Voter ID:server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb Address:682dc354-60da-e5d9-6e8a-d5a7aeba1fbb}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:20.892Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:20.892Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: 2025-02-09T16:52:20.892Z [INFO] server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: entering follower state: follower="Node at a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad [Follower]" leader= - testing.go:149: server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: 2025-02-09T16:52:20.892Z [INFO] server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad: entering follower state: follower="Node at a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad [Follower]" leader= -testLoggerAdapter verbose: server-74c08026-5219-806f-4c3e-149fcd4e57b9: 2025-02-09T16:52:20.893Z [INFO] server-74c08026-5219-806f-4c3e-149fcd4e57b9: entering follower state: follower="Node at 74c08026-5219-806f-4c3e-149fcd4e57b9 [Follower]" leader= - testing.go:149: server-74c08026-5219-806f-4c3e-149fcd4e57b9: 2025-02-09T16:52:20.893Z [INFO] server-74c08026-5219-806f-4c3e-149fcd4e57b9: entering follower state: follower="Node at 74c08026-5219-806f-4c3e-149fcd4e57b9 [Follower]" leader= -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.893Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering follower state: follower="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Follower]" leader= - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.893Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering follower state: follower="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Follower]" leader= -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [WARN] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [WARN] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering candidate state: node="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Candidate]" term=2 - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering candidate state: node="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Candidate]" term=2 -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: election won: tally=2 - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: election won: tally=2 -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering leader state: leader="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Leader]" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: entering leader state: leader="Node at 682dc354-60da-e5d9-6e8a-d5a7aeba1fbb [Leader]" -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: added peer, starting replication: peer=server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: added peer, starting replication: peer=server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: added peer, starting replication: peer=server-74c08026-5219-806f-4c3e-149fcd4e57b9 - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: added peer, starting replication: peer=server-74c08026-5219-806f-4c3e-149fcd4e57b9 -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: pipelining replication: peer="{Voter server-74c08026-5219-806f-4c3e-149fcd4e57b9 74c08026-5219-806f-4c3e-149fcd4e57b9}" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: pipelining replication: peer="{Voter server-74c08026-5219-806f-4c3e-149fcd4e57b9 74c08026-5219-806f-4c3e-149fcd4e57b9}" -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: pipelining replication: peer="{Voter server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad}" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:20.964Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: pipelining replication: peer="{Voter server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.070Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.964802621 +0000 UTC m=+15.295035309 (2 nodes), 71.819117ms from start of poll, 72.170302ms from cluster start. Timeout at 2025-02-09 16:52:21.070209449 +0000 UTC m=+15.400442137, 105.406828ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.070Z [INFO] stable state for Follower reached at 2025-02-09 16:52:20.964802621 +0000 UTC m=+15.295035309 (2 nodes), 71.819117ms from start of poll, 72.170302ms from cluster start. Timeout at 2025-02-09 16:52:21.070209449 +0000 UTC m=+15.400442137, 105.406828ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.070Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.070Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.175Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.070325701 +0000 UTC m=+15.400558387 (1 nodes), 886ns from start of poll, 177.69338ms from cluster start. Timeout at 2025-02-09 16:52:21.175768196 +0000 UTC m=+15.506000882, 105.442495ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.175Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.070325701 +0000 UTC m=+15.400558387 (1 nodes), 886ns from start of poll, 177.69338ms from cluster start. Timeout at 2025-02-09 16:52:21.175768196 +0000 UTC m=+15.506000882, 105.442495ms after stability -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:21.175Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: aborting pipeline replication: peer="{Voter server-74c08026-5219-806f-4c3e-149fcd4e57b9 74c08026-5219-806f-4c3e-149fcd4e57b9}" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:21.175Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: aborting pipeline replication: peer="{Voter server-74c08026-5219-806f-4c3e-149fcd4e57b9 74c08026-5219-806f-4c3e-149fcd4e57b9}" -testLoggerAdapter verbose: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:21.175Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: aborting pipeline replication: peer="{Voter server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad}" - testing.go:149: server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: 2025-02-09T16:52:21.175Z [INFO] server-682dc354-60da-e5d9-6e8a-d5a7aeba1fbb: aborting pipeline replication: peer="{Voter server-a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad a6a7d5ff-0239-4cd6-08fa-b27103f1f0ad}" ---- PASS: TestRaft_ProtocolVersion_RejectRPC (0.28s) +testLoggerAdapter verbose: server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: 2024-01-08T10:38:08.782Z [INFO] server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" + testing.go:149: server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: 2024-01-08T10:38:08.782Z [INFO] server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" +testLoggerAdapter verbose: server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: 2024-01-08T10:38:08.782Z [INFO] server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: entering follower state: follower="Node at 2fa038e7-29ef-37fc-95fb-c3d54d7ed694 [Follower]" leader= +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.783Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" + testing.go:149: server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: 2024-01-08T10:38:08.782Z [INFO] server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694: entering follower state: follower="Node at 2fa038e7-29ef-37fc-95fb-c3d54d7ed694 [Follower]" leader= + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.783Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.783Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering follower state: follower="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Follower]" leader= + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.783Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering follower state: follower="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Follower]" leader= +testLoggerAdapter verbose: server-a8652d3e-1813-2801-62af-3ceee35f1025: 2024-01-08T10:38:08.783Z [INFO] server-a8652d3e-1813-2801-62af-3ceee35f1025: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" + testing.go:149: server-a8652d3e-1813-2801-62af-3ceee35f1025: 2024-01-08T10:38:08.783Z [INFO] server-a8652d3e-1813-2801-62af-3ceee35f1025: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 Address:2fa038e7-29ef-37fc-95fb-c3d54d7ed694} {Suffrage:Voter ID:server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d Address:ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d} {Suffrage:Voter ID:server-a8652d3e-1813-2801-62af-3ceee35f1025 Address:a8652d3e-1813-2801-62af-3ceee35f1025}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.784Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:08.784Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-a8652d3e-1813-2801-62af-3ceee35f1025: 2024-01-08T10:38:08.784Z [INFO] server-a8652d3e-1813-2801-62af-3ceee35f1025: entering follower state: follower="Node at a8652d3e-1813-2801-62af-3ceee35f1025 [Follower]" leader= + testing.go:149: server-a8652d3e-1813-2801-62af-3ceee35f1025: 2024-01-08T10:38:08.784Z [INFO] server-a8652d3e-1813-2801-62af-3ceee35f1025: entering follower state: follower="Node at a8652d3e-1813-2801-62af-3ceee35f1025 [Follower]" leader= +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.858Z [WARN] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.858Z [WARN] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.858Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering candidate state: node="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Candidate]" term=2 + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.858Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering candidate state: node="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Candidate]" term=2 +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: election won: tally=2 + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: election won: tally=2 +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering leader state: leader="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Leader]" + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: entering leader state: leader="Node at ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d [Leader]" +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: added peer, starting replication: peer=server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: added peer, starting replication: peer=server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: added peer, starting replication: peer=server-a8652d3e-1813-2801-62af-3ceee35f1025 + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.859Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: added peer, starting replication: peer=server-a8652d3e-1813-2801-62af-3ceee35f1025 +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.860Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: pipelining replication: peer="{Voter server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 2fa038e7-29ef-37fc-95fb-c3d54d7ed694}" + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.860Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: pipelining replication: peer="{Voter server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 2fa038e7-29ef-37fc-95fb-c3d54d7ed694}" +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.860Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: pipelining replication: peer="{Voter server-a8652d3e-1813-2801-62af-3ceee35f1025 a8652d3e-1813-2801-62af-3ceee35f1025}" + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:08.860Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: pipelining replication: peer="{Voter server-a8652d3e-1813-2801-62af-3ceee35f1025 a8652d3e-1813-2801-62af-3ceee35f1025}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.966Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.859546107 +0000 UTC m=+17.011340244 (2 nodes), 75.008564ms from start of poll, 78.521275ms from cluster start. Timeout at 2024-01-08 10:38:08.965981199 +0000 UTC m=+17.117775316, 106.435072ms after stability + testing.go:149: cluster: 2024-01-08T10:38:08.966Z [INFO] stable state for Follower reached at 2024-01-08 10:38:08.859546107 +0000 UTC m=+17.011340244 (2 nodes), 75.008564ms from start of poll, 78.521275ms from cluster start. Timeout at 2024-01-08 10:38:08.965981199 +0000 UTC m=+17.117775316, 106.435072ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:08.966Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:08.966Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.072Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.966709538 +0000 UTC m=+17.118503654 (1 nodes), 3.166µs from start of poll, 185.684685ms from cluster start. Timeout at 2024-01-08 10:38:09.072692883 +0000 UTC m=+17.224487043, 105.983389ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.072Z [INFO] stable state for Leader reached at 2024-01-08 10:38:08.966709538 +0000 UTC m=+17.118503654 (1 nodes), 3.166µs from start of poll, 185.684685ms from cluster start. Timeout at 2024-01-08 10:38:09.072692883 +0000 UTC m=+17.224487043, 105.983389ms after stability +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:09.073Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: aborting pipeline replication: peer="{Voter server-a8652d3e-1813-2801-62af-3ceee35f1025 a8652d3e-1813-2801-62af-3ceee35f1025}" + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:09.073Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: aborting pipeline replication: peer="{Voter server-a8652d3e-1813-2801-62af-3ceee35f1025 a8652d3e-1813-2801-62af-3ceee35f1025}" +testLoggerAdapter verbose: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:09.073Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: aborting pipeline replication: peer="{Voter server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 2fa038e7-29ef-37fc-95fb-c3d54d7ed694}" + testing.go:149: server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: 2024-01-08T10:38:09.073Z [INFO] server-ef0b8cc6-1c5c-9595-a5c5-3ed6e7e8f45d: aborting pipeline replication: peer="{Voter server-2fa038e7-29ef-37fc-95fb-c3d54d7ed694 2fa038e7-29ef-37fc-95fb-c3d54d7ed694}" +--- PASS: TestRaft_ProtocolVersion_RejectRPC (0.30s) === RUN TestRaft_ProtocolVersion_Upgrade_1_2 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.176Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.176Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" -testLoggerAdapter verbose: 5d88de32-d2e8-e310-87e7-265905441d90: 2025-02-09T16:52:21.176Z [INFO] 5d88de32-d2e8-e310-87e7-265905441d90: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" - testing.go:149: 5d88de32-d2e8-e310-87e7-265905441d90: 2025-02-09T16:52:21.176Z [INFO] 5d88de32-d2e8-e310-87e7-265905441d90: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" -testLoggerAdapter verbose: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.176Z [INFO] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: initial configuration: index=0 servers=[] - testing.go:149: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.176Z [INFO] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.176Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.176Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.176Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering follower state: follower="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Follower]" leader= - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.176Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering follower state: follower="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Follower]" leader= -testLoggerAdapter verbose: 5d88de32-d2e8-e310-87e7-265905441d90: 2025-02-09T16:52:21.176Z [INFO] 5d88de32-d2e8-e310-87e7-265905441d90: entering follower state: follower="Node at 5d88de32-d2e8-e310-87e7-265905441d90 [Follower]" leader= - testing.go:149: 5d88de32-d2e8-e310-87e7-265905441d90: 2025-02-09T16:52:21.176Z [INFO] 5d88de32-d2e8-e310-87e7-265905441d90: entering follower state: follower="Node at 5d88de32-d2e8-e310-87e7-265905441d90 [Follower]" leader= -testLoggerAdapter verbose: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.176Z [INFO] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: entering follower state: follower="Node at 7f7cbfad-545e-d2f6-2461-dab52d3c9988 [Follower]" leader= - testing.go:149: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.176Z [INFO] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: entering follower state: follower="Node at 7f7cbfad-545e-d2f6-2461-dab52d3c9988 [Follower]" leader= -testLoggerAdapter verbose: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.255Z [WARN] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: no known peers, aborting election - testing.go:149: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.255Z [WARN] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: no known peers, aborting election -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [WARN] c7bdc695-601c-95ba-9197-d6189509ca62: heartbeat timeout reached, starting election: last-leader= - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [WARN] c7bdc695-601c-95ba-9197-d6189509ca62: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering candidate state: node="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Candidate]" term=2 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering candidate state: node="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Candidate]" term=2 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: election won: tally=2 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: election won: tally=2 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering leader state: leader="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Leader]" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: entering leader state: leader="Node at c7bdc695-601c-95ba-9197-d6189509ca62 [Leader]" -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: added peer, starting replication: peer=5d88de32-d2e8-e310-87e7-265905441d90 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: added peer, starting replication: peer=5d88de32-d2e8-e310-87e7-265905441d90 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: pipelining replication: peer="{Voter 5d88de32-d2e8-e310-87e7-265905441d90 5d88de32-d2e8-e310-87e7-265905441d90}" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.261Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: pipelining replication: peer="{Voter 5d88de32-d2e8-e310-87e7-265905441d90 5d88de32-d2e8-e310-87e7-265905441d90}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.366Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.261505847 +0000 UTC m=+15.591738535 (1 nodes), 84.715585ms from start of poll, 85.062224ms from cluster start. Timeout at 2025-02-09 16:52:21.366708262 +0000 UTC m=+15.696940950, 105.202415ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.366Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.261505847 +0000 UTC m=+15.591738535 (1 nodes), 84.715585ms from start of poll, 85.062224ms from cluster start. Timeout at 2025-02-09 16:52:21.366708262 +0000 UTC m=+15.696940950, 105.202415ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.366Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.366Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.472Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.366839807 +0000 UTC m=+15.697072493 (1 nodes), 1.115µs from start of poll, 190.396182ms from cluster start. Timeout at 2025-02-09 16:52:21.472255637 +0000 UTC m=+15.802488325, 105.415832ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.472Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.366839807 +0000 UTC m=+15.697072493 (1 nodes), 1.115µs from start of poll, 190.396182ms from cluster start. Timeout at 2025-02-09 16:52:21.472255637 +0000 UTC m=+15.802488325, 105.415832ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.472Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.472Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.577Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.472371037 +0000 UTC m=+15.802603723 (1 nodes), 696ns from start of poll, 295.927412ms from cluster start. Timeout at 2025-02-09 16:52:21.577864808 +0000 UTC m=+15.908097495, 105.493772ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.577Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.472371037 +0000 UTC m=+15.802603723 (1 nodes), 696ns from start of poll, 295.927412ms from cluster start. Timeout at 2025-02-09 16:52:21.577864808 +0000 UTC m=+15.908097495, 105.493772ms after stability -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.577Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: updating configuration: command=AddStaging server-id=7f7cbfad-545e-d2f6-2461-dab52d3c9988 server-addr=7f7cbfad-545e-d2f6-2461-dab52d3c9988 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90} {Suffrage:Voter ID:7f7cbfad-545e-d2f6-2461-dab52d3c9988 Address:7f7cbfad-545e-d2f6-2461-dab52d3c9988}]" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.577Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: updating configuration: command=AddStaging server-id=7f7cbfad-545e-d2f6-2461-dab52d3c9988 server-addr=7f7cbfad-545e-d2f6-2461-dab52d3c9988 servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90} {Suffrage:Voter ID:7f7cbfad-545e-d2f6-2461-dab52d3c9988 Address:7f7cbfad-545e-d2f6-2461-dab52d3c9988}]" -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: added peer, starting replication: peer=7f7cbfad-545e-d2f6-2461-dab52d3c9988 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: added peer, starting replication: peer=7f7cbfad-545e-d2f6-2461-dab52d3c9988 -testLoggerAdapter verbose: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.578Z [WARN] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: 7f7cbfad-545e-d2f6-2461-dab52d3c9988: 2025-02-09T16:52:21.578Z [WARN] 7f7cbfad-545e-d2f6-2461-dab52d3c9988: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [WARN] c7bdc695-601c-95ba-9197-d6189509ca62: appendEntries rejected, sending older logs: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" next=1 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [WARN] c7bdc695-601c-95ba-9197-d6189509ca62: appendEntries rejected, sending older logs: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" next=1 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: pipelining replication: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.578Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: pipelining replication: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.578Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.578Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.683Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.578330442 +0000 UTC m=+15.908563128 (1 nodes), 488ns from start of poll, 401.886817ms from cluster start. Timeout at 2025-02-09 16:52:21.683412841 +0000 UTC m=+16.013645527, 105.082399ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.683Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.578330442 +0000 UTC m=+15.908563128 (1 nodes), 488ns from start of poll, 401.886817ms from cluster start. Timeout at 2025-02-09 16:52:21.683412841 +0000 UTC m=+16.013645527, 105.082399ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.683Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.683Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.789Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.683538674 +0000 UTC m=+16.013771361 (1 nodes), 606ns from start of poll, 507.09505ms from cluster start. Timeout at 2025-02-09 16:52:21.78906047 +0000 UTC m=+16.119293159, 105.521798ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.789Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.683538674 +0000 UTC m=+16.013771361 (1 nodes), 606ns from start of poll, 507.09505ms from cluster start. Timeout at 2025-02-09 16:52:21.78906047 +0000 UTC m=+16.119293159, 105.521798ms after stability -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: updating configuration: command=RemoveServer server-id=7f7cbfad-545e-d2f6-2461-dab52d3c9988 server-addr= servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: updating configuration: command=RemoveServer server-id=7f7cbfad-545e-d2f6-2461-dab52d3c9988 server-addr= servers="[{Suffrage:Voter ID:c7bdc695-601c-95ba-9197-d6189509ca62 Address:c7bdc695-601c-95ba-9197-d6189509ca62} {Suffrage:Voter ID:5d88de32-d2e8-e310-87e7-265905441d90 Address:5d88de32-d2e8-e310-87e7-265905441d90}]" -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: removed peer, stopping replication: peer=7f7cbfad-545e-d2f6-2461-dab52d3c9988 last-index=4 - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: removed peer, stopping replication: peer=7f7cbfad-545e-d2f6-2461-dab52d3c9988 last-index=4 -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: aborting pipeline replication: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: aborting pipeline replication: peer="{Voter 7f7cbfad-545e-d2f6-2461-dab52d3c9988 7f7cbfad-545e-d2f6-2461-dab52d3c9988}" -testLoggerAdapter verbose: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: aborting pipeline replication: peer="{Voter 5d88de32-d2e8-e310-87e7-265905441d90 5d88de32-d2e8-e310-87e7-265905441d90}" - testing.go:149: c7bdc695-601c-95ba-9197-d6189509ca62: 2025-02-09T16:52:21.789Z [INFO] c7bdc695-601c-95ba-9197-d6189509ca62: aborting pipeline replication: peer="{Voter 5d88de32-d2e8-e310-87e7-265905441d90 5d88de32-d2e8-e310-87e7-265905441d90}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_1_2 (0.61s) +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.078Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: initial configuration: index=1 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.078Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: initial configuration: index=1 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" +testLoggerAdapter verbose: 1cac2c58-f503-a02c-4055-81f1d9ef4b97: 2024-01-08T10:38:09.079Z [INFO] 1cac2c58-f503-a02c-4055-81f1d9ef4b97: initial configuration: index=1 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" + testing.go:149: 1cac2c58-f503-a02c-4055-81f1d9ef4b97: 2024-01-08T10:38:09.079Z [INFO] 1cac2c58-f503-a02c-4055-81f1d9ef4b97: initial configuration: index=1 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.079Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering follower state: follower="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Follower]" leader= + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.079Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering follower state: follower="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Follower]" leader= +testLoggerAdapter verbose: 1cac2c58-f503-a02c-4055-81f1d9ef4b97: 2024-01-08T10:38:09.079Z [INFO] 1cac2c58-f503-a02c-4055-81f1d9ef4b97: entering follower state: follower="Node at 1cac2c58-f503-a02c-4055-81f1d9ef4b97 [Follower]" leader= +testLoggerAdapter verbose: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.079Z [INFO] 274ff229-d96c-710d-285b-11d1c2b0f378: initial configuration: index=0 servers=[] + testing.go:149: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.079Z [INFO] 274ff229-d96c-710d-285b-11d1c2b0f378: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.080Z [INFO] starting stability test: raft-state=Leader + testing.go:149: 1cac2c58-f503-a02c-4055-81f1d9ef4b97: 2024-01-08T10:38:09.079Z [INFO] 1cac2c58-f503-a02c-4055-81f1d9ef4b97: entering follower state: follower="Node at 1cac2c58-f503-a02c-4055-81f1d9ef4b97 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T10:38:09.080Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.080Z [INFO] 274ff229-d96c-710d-285b-11d1c2b0f378: entering follower state: follower="Node at 274ff229-d96c-710d-285b-11d1c2b0f378 [Follower]" leader= + testing.go:149: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.080Z [INFO] 274ff229-d96c-710d-285b-11d1c2b0f378: entering follower state: follower="Node at 274ff229-d96c-710d-285b-11d1c2b0f378 [Follower]" leader= +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.146Z [WARN] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: heartbeat timeout reached, starting election: last-leader= + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.146Z [WARN] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.146Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering candidate state: node="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Candidate]" term=2 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.146Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering candidate state: node="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Candidate]" term=2 +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.147Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: election won: tally=2 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.147Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: election won: tally=2 +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.147Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering leader state: leader="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Leader]" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.147Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: entering leader state: leader="Node at 42460d72-c68b-3f8e-98b1-a7c039f9d77c [Leader]" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.148Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: added peer, starting replication: peer=1cac2c58-f503-a02c-4055-81f1d9ef4b97 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.148Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: added peer, starting replication: peer=1cac2c58-f503-a02c-4055-81f1d9ef4b97 +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.148Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: pipelining replication: peer="{Voter 1cac2c58-f503-a02c-4055-81f1d9ef4b97 1cac2c58-f503-a02c-4055-81f1d9ef4b97}" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.148Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: pipelining replication: peer="{Voter 1cac2c58-f503-a02c-4055-81f1d9ef4b97 1cac2c58-f503-a02c-4055-81f1d9ef4b97}" +testLoggerAdapter verbose: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.165Z [WARN] 274ff229-d96c-710d-285b-11d1c2b0f378: no known peers, aborting election + testing.go:149: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.165Z [WARN] 274ff229-d96c-710d-285b-11d1c2b0f378: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.254Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.148436747 +0000 UTC m=+17.300230921 (1 nodes), 67.835869ms from start of poll, 71.209507ms from cluster start. Timeout at 2024-01-08 10:38:09.254597882 +0000 UTC m=+17.406392004, 106.161083ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.254Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.148436747 +0000 UTC m=+17.300230921 (1 nodes), 67.835869ms from start of poll, 71.209507ms from cluster start. Timeout at 2024-01-08 10:38:09.254597882 +0000 UTC m=+17.406392004, 106.161083ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.255Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.255Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.360Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.255349213 +0000 UTC m=+17.407143318 (1 nodes), 3.715µs from start of poll, 178.121904ms from cluster start. Timeout at 2024-01-08 10:38:09.360406708 +0000 UTC m=+17.512200846, 105.057528ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.360Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.255349213 +0000 UTC m=+17.407143318 (1 nodes), 3.715µs from start of poll, 178.121904ms from cluster start. Timeout at 2024-01-08 10:38:09.360406708 +0000 UTC m=+17.512200846, 105.057528ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.361Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.361Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.470Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.361353918 +0000 UTC m=+17.513148033 (1 nodes), 3.798µs from start of poll, 284.126619ms from cluster start. Timeout at 2024-01-08 10:38:09.469918441 +0000 UTC m=+17.621712589, 108.564556ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.470Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.361353918 +0000 UTC m=+17.513148033 (1 nodes), 3.798µs from start of poll, 284.126619ms from cluster start. Timeout at 2024-01-08 10:38:09.469918441 +0000 UTC m=+17.621712589, 108.564556ms after stability +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.470Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: updating configuration: command=AddStaging server-id=274ff229-d96c-710d-285b-11d1c2b0f378 server-addr=274ff229-d96c-710d-285b-11d1c2b0f378 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97} {Suffrage:Voter ID:274ff229-d96c-710d-285b-11d1c2b0f378 Address:274ff229-d96c-710d-285b-11d1c2b0f378}]" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.470Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: updating configuration: command=AddStaging server-id=274ff229-d96c-710d-285b-11d1c2b0f378 server-addr=274ff229-d96c-710d-285b-11d1c2b0f378 servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97} {Suffrage:Voter ID:274ff229-d96c-710d-285b-11d1c2b0f378 Address:274ff229-d96c-710d-285b-11d1c2b0f378}]" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.471Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: added peer, starting replication: peer=274ff229-d96c-710d-285b-11d1c2b0f378 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.471Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: added peer, starting replication: peer=274ff229-d96c-710d-285b-11d1c2b0f378 +testLoggerAdapter verbose: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.472Z [WARN] 274ff229-d96c-710d-285b-11d1c2b0f378: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: 274ff229-d96c-710d-285b-11d1c2b0f378: 2024-01-08T10:38:09.472Z [WARN] 274ff229-d96c-710d-285b-11d1c2b0f378: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.472Z [WARN] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: appendEntries rejected, sending older logs: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" next=1 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.472Z [WARN] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: appendEntries rejected, sending older logs: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" next=1 +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.472Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: pipelining replication: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.472Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: pipelining replication: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.478Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.478Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.584Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.478355191 +0000 UTC m=+17.630149315 (1 nodes), 3.162µs from start of poll, 401.127901ms from cluster start. Timeout at 2024-01-08 10:38:09.584498811 +0000 UTC m=+17.736292949, 106.143634ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.584Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.478355191 +0000 UTC m=+17.630149315 (1 nodes), 3.162µs from start of poll, 401.127901ms from cluster start. Timeout at 2024-01-08 10:38:09.584498811 +0000 UTC m=+17.736292949, 106.143634ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.584Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.584Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.692Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.585648147 +0000 UTC m=+17.737442271 (1 nodes), 3.755µs from start of poll, 508.420857ms from cluster start. Timeout at 2024-01-08 10:38:09.691920519 +0000 UTC m=+17.843714653, 106.272382ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.692Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.585648147 +0000 UTC m=+17.737442271 (1 nodes), 3.755µs from start of poll, 508.420857ms from cluster start. Timeout at 2024-01-08 10:38:09.691920519 +0000 UTC m=+17.843714653, 106.272382ms after stability +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.692Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: updating configuration: command=RemoveServer server-id=274ff229-d96c-710d-285b-11d1c2b0f378 server-addr= servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.692Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: updating configuration: command=RemoveServer server-id=274ff229-d96c-710d-285b-11d1c2b0f378 server-addr= servers="[{Suffrage:Voter ID:42460d72-c68b-3f8e-98b1-a7c039f9d77c Address:42460d72-c68b-3f8e-98b1-a7c039f9d77c} {Suffrage:Voter ID:1cac2c58-f503-a02c-4055-81f1d9ef4b97 Address:1cac2c58-f503-a02c-4055-81f1d9ef4b97}]" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.692Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: removed peer, stopping replication: peer=274ff229-d96c-710d-285b-11d1c2b0f378 last-index=4 + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.692Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: removed peer, stopping replication: peer=274ff229-d96c-710d-285b-11d1c2b0f378 last-index=4 +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.693Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: aborting pipeline replication: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.693Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: aborting pipeline replication: peer="{Voter 274ff229-d96c-710d-285b-11d1c2b0f378 274ff229-d96c-710d-285b-11d1c2b0f378}" +testLoggerAdapter verbose: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.693Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: aborting pipeline replication: peer="{Voter 1cac2c58-f503-a02c-4055-81f1d9ef4b97 1cac2c58-f503-a02c-4055-81f1d9ef4b97}" + testing.go:149: 42460d72-c68b-3f8e-98b1-a7c039f9d77c: 2024-01-08T10:38:09.693Z [INFO] 42460d72-c68b-3f8e-98b1-a7c039f9d77c: aborting pipeline replication: peer="{Voter 1cac2c58-f503-a02c-4055-81f1d9ef4b97 1cac2c58-f503-a02c-4055-81f1d9ef4b97}" +--- PASS: TestRaft_ProtocolVersion_Upgrade_1_2 (0.62s) === RUN TestRaft_ProtocolVersion_Upgrade_2_3 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.790Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0}]" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.790Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0}]" -testLoggerAdapter verbose: 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: 2025-02-09T16:52:21.790Z [INFO] 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0}]" - testing.go:149: 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: 2025-02-09T16:52:21.790Z [INFO] 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.790Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:21.790Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.790Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering follower state: follower="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Follower]" leader= - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.790Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering follower state: follower="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Follower]" leader= -testLoggerAdapter verbose: 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: 2025-02-09T16:52:21.790Z [INFO] 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: entering follower state: follower="Node at 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 [Follower]" leader= - testing.go:149: 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: 2025-02-09T16:52:21.790Z [INFO] 1bf2144f-27b6-25a8-ee10-8bad08cef5b0: entering follower state: follower="Node at 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 [Follower]" leader= -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [WARN] cb71177c-0d9d-05c2-08c5-89187a942a57: heartbeat timeout reached, starting election: last-leader= - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [WARN] cb71177c-0d9d-05c2-08c5-89187a942a57: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering candidate state: node="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Candidate]" term=2 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering candidate state: node="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Candidate]" term=2 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: election won: tally=2 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: election won: tally=2 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering leader state: leader="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Leader]" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: entering leader state: leader="Node at cb71177c-0d9d-05c2-08c5-89187a942a57 [Leader]" -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: added peer, starting replication: peer=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: added peer, starting replication: peer=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: pipelining replication: peer="{Voter 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 1bf2144f-27b6-25a8-ee10-8bad08cef5b0}" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:21.876Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: pipelining replication: peer="{Voter 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 1bf2144f-27b6-25a8-ee10-8bad08cef5b0}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.982Z [INFO] stable state for Follower reached at 2025-02-09 16:52:21.876704244 +0000 UTC m=+16.206936932 (1 nodes), 86.55947ms from start of poll, 86.772246ms from cluster start. Timeout at 2025-02-09 16:52:21.982517527 +0000 UTC m=+16.312750213, 105.813281ms after stability - testing.go:149: cluster: 2025-02-09T16:52:21.982Z [INFO] stable state for Follower reached at 2025-02-09 16:52:21.876704244 +0000 UTC m=+16.206936932 (1 nodes), 86.55947ms from start of poll, 86.772246ms from cluster start. Timeout at 2025-02-09 16:52:21.982517527 +0000 UTC m=+16.312750213, 105.813281ms after stability -testLoggerAdapter verbose: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:21.982Z [INFO] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: initial configuration: index=0 servers=[] - testing.go:149: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:21.982Z [INFO] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: initial configuration: index=0 servers=[] -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:21.982Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:21.982Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:21.982Z [INFO] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: entering follower state: follower="Node at 876eab3e-8c07-6e16-5504-5b31fb6774d7 [Follower]" leader= - testing.go:149: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:21.982Z [INFO] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: entering follower state: follower="Node at 876eab3e-8c07-6e16-5504-5b31fb6774d7 [Follower]" leader= -testLoggerAdapter verbose: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:22.068Z [WARN] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: no known peers, aborting election - testing.go:149: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:22.068Z [WARN] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: no known peers, aborting election -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.088Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.982872851 +0000 UTC m=+16.313105538 (1 nodes), 504ns from start of poll, 192.940852ms from cluster start. Timeout at 2025-02-09 16:52:22.088616253 +0000 UTC m=+16.418848941, 105.743403ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.088Z [INFO] stable state for Leader reached at 2025-02-09 16:52:21.982872851 +0000 UTC m=+16.313105538 (1 nodes), 504ns from start of poll, 192.940852ms from cluster start. Timeout at 2025-02-09 16:52:22.088616253 +0000 UTC m=+16.418848941, 105.743403ms after stability -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: updating configuration: command=AddStaging server-id=server-876eab3e-8c07-6e16-5504-5b31fb6774d7 server-addr=876eab3e-8c07-6e16-5504-5b31fb6774d7 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0} {Suffrage:Voter ID:server-876eab3e-8c07-6e16-5504-5b31fb6774d7 Address:876eab3e-8c07-6e16-5504-5b31fb6774d7}]" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: updating configuration: command=AddStaging server-id=server-876eab3e-8c07-6e16-5504-5b31fb6774d7 server-addr=876eab3e-8c07-6e16-5504-5b31fb6774d7 servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:1bf2144f-27b6-25a8-ee10-8bad08cef5b0 Address:1bf2144f-27b6-25a8-ee10-8bad08cef5b0} {Suffrage:Voter ID:server-876eab3e-8c07-6e16-5504-5b31fb6774d7 Address:876eab3e-8c07-6e16-5504-5b31fb6774d7}]" -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: added peer, starting replication: peer=server-876eab3e-8c07-6e16-5504-5b31fb6774d7 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: added peer, starting replication: peer=server-876eab3e-8c07-6e16-5504-5b31fb6774d7 -testLoggerAdapter verbose: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:22.088Z [WARN] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: failed to get previous log: previous-index=3 last-index=0 error="log not found" - testing.go:149: server-876eab3e-8c07-6e16-5504-5b31fb6774d7: 2025-02-09T16:52:22.088Z [WARN] server-876eab3e-8c07-6e16-5504-5b31fb6774d7: failed to get previous log: previous-index=3 last-index=0 error="log not found" -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [WARN] cb71177c-0d9d-05c2-08c5-89187a942a57: appendEntries rejected, sending older logs: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" next=1 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [WARN] cb71177c-0d9d-05c2-08c5-89187a942a57: appendEntries rejected, sending older logs: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" next=1 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: pipelining replication: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.088Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: pipelining replication: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.089Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.089Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.194Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.089091286 +0000 UTC m=+16.419323972 (1 nodes), 490ns from start of poll, 299.159286ms from cluster start. Timeout at 2025-02-09 16:52:22.194761208 +0000 UTC m=+16.524993893, 105.669921ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.194Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.089091286 +0000 UTC m=+16.419323972 (1 nodes), 490ns from start of poll, 299.159286ms from cluster start. Timeout at 2025-02-09 16:52:22.194761208 +0000 UTC m=+16.524993893, 105.669921ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.194Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.194Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.300Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.194890141 +0000 UTC m=+16.525122826 (1 nodes), 690ns from start of poll, 404.95814ms from cluster start. Timeout at 2025-02-09 16:52:22.30064535 +0000 UTC m=+16.630878038, 105.755212ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.300Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.194890141 +0000 UTC m=+16.525122826 (1 nodes), 690ns from start of poll, 404.95814ms from cluster start. Timeout at 2025-02-09 16:52:22.30064535 +0000 UTC m=+16.630878038, 105.755212ms after stability -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: updating configuration: command=RemoveServer server-id=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 server-addr= servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:server-876eab3e-8c07-6e16-5504-5b31fb6774d7 Address:876eab3e-8c07-6e16-5504-5b31fb6774d7}]" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: updating configuration: command=RemoveServer server-id=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 server-addr= servers="[{Suffrage:Voter ID:cb71177c-0d9d-05c2-08c5-89187a942a57 Address:cb71177c-0d9d-05c2-08c5-89187a942a57} {Suffrage:Voter ID:server-876eab3e-8c07-6e16-5504-5b31fb6774d7 Address:876eab3e-8c07-6e16-5504-5b31fb6774d7}]" -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: removed peer, stopping replication: peer=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 last-index=4 - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: removed peer, stopping replication: peer=1bf2144f-27b6-25a8-ee10-8bad08cef5b0 last-index=4 -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: aborting pipeline replication: peer="{Voter 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 1bf2144f-27b6-25a8-ee10-8bad08cef5b0}" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: aborting pipeline replication: peer="{Voter 1bf2144f-27b6-25a8-ee10-8bad08cef5b0 1bf2144f-27b6-25a8-ee10-8bad08cef5b0}" -testLoggerAdapter verbose: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: aborting pipeline replication: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" - testing.go:149: cb71177c-0d9d-05c2-08c5-89187a942a57: 2025-02-09T16:52:22.300Z [INFO] cb71177c-0d9d-05c2-08c5-89187a942a57: aborting pipeline replication: peer="{Voter server-876eab3e-8c07-6e16-5504-5b31fb6774d7 876eab3e-8c07-6e16-5504-5b31fb6774d7}" ---- PASS: TestRaft_ProtocolVersion_Upgrade_2_3 (0.51s) +testLoggerAdapter verbose: 72508752-429a-0774-8739-0022d01709dc: 2024-01-08T10:38:09.698Z [INFO] 72508752-429a-0774-8739-0022d01709dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697}]" + testing.go:149: 72508752-429a-0774-8739-0022d01709dc: 2024-01-08T10:38:09.698Z [INFO] 72508752-429a-0774-8739-0022d01709dc: initial configuration: index=1 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697}]" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.698Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: initial configuration: index=1 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697}]" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.698Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: initial configuration: index=1 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.699Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:09.699Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.699Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering follower state: follower="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Follower]" leader= + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.699Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering follower state: follower="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Follower]" leader= +testLoggerAdapter verbose: 72508752-429a-0774-8739-0022d01709dc: 2024-01-08T10:38:09.699Z [INFO] 72508752-429a-0774-8739-0022d01709dc: entering follower state: follower="Node at 72508752-429a-0774-8739-0022d01709dc [Follower]" leader= + testing.go:149: 72508752-429a-0774-8739-0022d01709dc: 2024-01-08T10:38:09.699Z [INFO] 72508752-429a-0774-8739-0022d01709dc: entering follower state: follower="Node at 72508752-429a-0774-8739-0022d01709dc [Follower]" leader= +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.772Z [WARN] e8840dec-fdb5-2092-7b67-febd91c76697: heartbeat timeout reached, starting election: last-leader= + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.772Z [WARN] e8840dec-fdb5-2092-7b67-febd91c76697: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.773Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering candidate state: node="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Candidate]" term=2 + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.773Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering candidate state: node="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Candidate]" term=2 +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.773Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: election won: tally=2 + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.773Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: election won: tally=2 +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering leader state: leader="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Leader]" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: entering leader state: leader="Node at e8840dec-fdb5-2092-7b67-febd91c76697 [Leader]" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: added peer, starting replication: peer=72508752-429a-0774-8739-0022d01709dc + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: added peer, starting replication: peer=72508752-429a-0774-8739-0022d01709dc +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: pipelining replication: peer="{Voter 72508752-429a-0774-8739-0022d01709dc 72508752-429a-0774-8739-0022d01709dc}" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.774Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: pipelining replication: peer="{Voter 72508752-429a-0774-8739-0022d01709dc 72508752-429a-0774-8739-0022d01709dc}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.882Z [INFO] stable state for Follower reached at 2024-01-08 10:38:09.775223536 +0000 UTC m=+17.927017694 (1 nodes), 76.00369ms from start of poll, 77.658646ms from cluster start. Timeout at 2024-01-08 10:38:09.882139632 +0000 UTC m=+18.033933744, 106.91605ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.882Z [INFO] stable state for Follower reached at 2024-01-08 10:38:09.775223536 +0000 UTC m=+17.927017694 (1 nodes), 76.00369ms from start of poll, 77.658646ms from cluster start. Timeout at 2024-01-08 10:38:09.882139632 +0000 UTC m=+18.033933744, 106.91605ms after stability +testLoggerAdapter verbose: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.883Z [INFO] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: initial configuration: index=0 servers=[] + testing.go:149: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.883Z [INFO] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: initial configuration: index=0 servers=[] +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.884Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.884Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.884Z [INFO] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: entering follower state: follower="Node at 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 [Follower]" leader= + testing.go:149: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.884Z [INFO] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: entering follower state: follower="Node at 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 [Follower]" leader= +testLoggerAdapter verbose: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.986Z [WARN] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: no known peers, aborting election + testing.go:149: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.986Z [WARN] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: no known peers, aborting election +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.990Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.884434755 +0000 UTC m=+18.036228869 (1 nodes), 2.736µs from start of poll, 186.869821ms from cluster start. Timeout at 2024-01-08 10:38:09.990604844 +0000 UTC m=+18.142398962, 106.170093ms after stability + testing.go:149: cluster: 2024-01-08T10:38:09.990Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.884434755 +0000 UTC m=+18.036228869 (1 nodes), 2.736µs from start of poll, 186.869821ms from cluster start. Timeout at 2024-01-08 10:38:09.990604844 +0000 UTC m=+18.142398962, 106.170093ms after stability +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.991Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: updating configuration: command=AddStaging server-id=server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 server-addr=6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697} {Suffrage:Voter ID:server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 Address:6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}]" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.991Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: updating configuration: command=AddStaging server-id=server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 server-addr=6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 servers="[{Suffrage:Voter ID:72508752-429a-0774-8739-0022d01709dc Address:72508752-429a-0774-8739-0022d01709dc} {Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697} {Suffrage:Voter ID:server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 Address:6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}]" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.991Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: added peer, starting replication: peer=server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.991Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: added peer, starting replication: peer=server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 +testLoggerAdapter verbose: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.992Z [WARN] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: failed to get previous log: previous-index=3 last-index=0 error="log not found" + testing.go:149: server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: 2024-01-08T10:38:09.992Z [WARN] server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2: failed to get previous log: previous-index=3 last-index=0 error="log not found" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.992Z [WARN] e8840dec-fdb5-2092-7b67-febd91c76697: appendEntries rejected, sending older logs: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" next=1 + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.992Z [WARN] e8840dec-fdb5-2092-7b67-febd91c76697: appendEntries rejected, sending older logs: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" next=1 +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.993Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: pipelining replication: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:09.993Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: pipelining replication: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:09.998Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:09.998Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.104Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.998733888 +0000 UTC m=+18.150528008 (1 nodes), 3.263µs from start of poll, 301.16896ms from cluster start. Timeout at 2024-01-08 10:38:10.103791628 +0000 UTC m=+18.255585742, 105.057734ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.104Z [INFO] stable state for Leader reached at 2024-01-08 10:38:09.998733888 +0000 UTC m=+18.150528008 (1 nodes), 3.263µs from start of poll, 301.16896ms from cluster start. Timeout at 2024-01-08 10:38:10.103791628 +0000 UTC m=+18.255585742, 105.057734ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.104Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.104Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.210Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.104608976 +0000 UTC m=+18.256403102 (1 nodes), 3.794µs from start of poll, 407.044054ms from cluster start. Timeout at 2024-01-08 10:38:10.210444407 +0000 UTC m=+18.362238554, 105.835452ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.210Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.104608976 +0000 UTC m=+18.256403102 (1 nodes), 3.794µs from start of poll, 407.044054ms from cluster start. Timeout at 2024-01-08 10:38:10.210444407 +0000 UTC m=+18.362238554, 105.835452ms after stability +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.210Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: updating configuration: command=RemoveServer server-id=72508752-429a-0774-8739-0022d01709dc server-addr= servers="[{Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697} {Suffrage:Voter ID:server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 Address:6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}]" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.210Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: updating configuration: command=RemoveServer server-id=72508752-429a-0774-8739-0022d01709dc server-addr= servers="[{Suffrage:Voter ID:e8840dec-fdb5-2092-7b67-febd91c76697 Address:e8840dec-fdb5-2092-7b67-febd91c76697} {Suffrage:Voter ID:server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 Address:6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}]" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.211Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: removed peer, stopping replication: peer=72508752-429a-0774-8739-0022d01709dc last-index=4 + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.211Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: removed peer, stopping replication: peer=72508752-429a-0774-8739-0022d01709dc last-index=4 +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.212Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: aborting pipeline replication: peer="{Voter 72508752-429a-0774-8739-0022d01709dc 72508752-429a-0774-8739-0022d01709dc}" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.212Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: aborting pipeline replication: peer="{Voter 72508752-429a-0774-8739-0022d01709dc 72508752-429a-0774-8739-0022d01709dc}" +testLoggerAdapter verbose: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.212Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: aborting pipeline replication: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" + testing.go:149: e8840dec-fdb5-2092-7b67-febd91c76697: 2024-01-08T10:38:10.212Z [INFO] e8840dec-fdb5-2092-7b67-febd91c76697: aborting pipeline replication: peer="{Voter server-6913f6f8-c7e8-4e79-16ff-cd86e8a571a2 6913f6f8-c7e8-4e79-16ff-cd86e8a571a2}" +--- PASS: TestRaft_ProtocolVersion_Upgrade_2_3 (0.53s) === RUN TestRaft_LeadershipTransferInProgress --- PASS: TestRaft_LeadershipTransferInProgress (0.00s) === RUN TestRaft_LeadershipTransferPickServer --- PASS: TestRaft_LeadershipTransferPickServer (0.00s) === RUN TestRaft_LeadershipTransfer -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.301Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.301Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.301Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.301Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.301Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering follower state: follower="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Follower]" leader= -testLoggerAdapter verbose: server-bde78719-276c-0151-d7d9-2f2d65f22669: 2025-02-09T16:52:22.301Z [INFO] server-bde78719-276c-0151-d7d9-2f2d65f22669: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.301Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering follower state: follower="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Follower]" leader= -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.302Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering follower state: follower="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Follower]" leader= - testing.go:149: server-bde78719-276c-0151-d7d9-2f2d65f22669: 2025-02-09T16:52:22.301Z [INFO] server-bde78719-276c-0151-d7d9-2f2d65f22669: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-daf78732-292a-d894-97bb-bc3d80cb2891 Address:daf78732-292a-d894-97bb-bc3d80cb2891} {Suffrage:Voter ID:server-8ed6a849-50ac-04da-d74b-8e21f4edc48e Address:8ed6a849-50ac-04da-d74b-8e21f4edc48e} {Suffrage:Voter ID:server-bde78719-276c-0151-d7d9-2f2d65f22669 Address:bde78719-276c-0151-d7d9-2f2d65f22669}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.302Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.302Z [INFO] starting stability test: raft-state=Leader - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.302Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering follower state: follower="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Follower]" leader= -testLoggerAdapter verbose: server-bde78719-276c-0151-d7d9-2f2d65f22669: 2025-02-09T16:52:22.302Z [INFO] server-bde78719-276c-0151-d7d9-2f2d65f22669: entering follower state: follower="Node at bde78719-276c-0151-d7d9-2f2d65f22669 [Follower]" leader= - testing.go:149: server-bde78719-276c-0151-d7d9-2f2d65f22669: 2025-02-09T16:52:22.302Z [INFO] server-bde78719-276c-0151-d7d9-2f2d65f22669: entering follower state: follower="Node at bde78719-276c-0151-d7d9-2f2d65f22669 [Follower]" leader= -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [WARN] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [WARN] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering candidate state: node="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Candidate]" term=2 - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering candidate state: node="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Candidate]" term=2 -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: election won: tally=2 - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: election won: tally=2 -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering leader state: leader="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Leader]" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering leader state: leader="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Leader]" -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: added peer, starting replication: peer=server-daf78732-292a-d894-97bb-bc3d80cb2891 - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: added peer, starting replication: peer=server-daf78732-292a-d894-97bb-bc3d80cb2891 -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: added peer, starting replication: peer=server-bde78719-276c-0151-d7d9-2f2d65f22669 - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: added peer, starting replication: peer=server-bde78719-276c-0151-d7d9-2f2d65f22669 -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: pipelining replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.369Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: pipelining replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.370Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: pipelining replication: peer="{Voter server-daf78732-292a-d894-97bb-bc3d80cb2891 daf78732-292a-d894-97bb-bc3d80cb2891}" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.370Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: pipelining replication: peer="{Voter server-daf78732-292a-d894-97bb-bc3d80cb2891 daf78732-292a-d894-97bb-bc3d80cb2891}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.475Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.370133162 +0000 UTC m=+16.700365850 (1 nodes), 68.050621ms from start of poll, 68.543527ms from cluster start. Timeout at 2025-02-09 16:52:22.475258189 +0000 UTC m=+16.805490877, 105.125027ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.475Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.370133162 +0000 UTC m=+16.700365850 (1 nodes), 68.050621ms from start of poll, 68.543527ms from cluster start. Timeout at 2025-02-09 16:52:22.475258189 +0000 UTC m=+16.805490877, 105.125027ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.475Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.475Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.581Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.475374419 +0000 UTC m=+16.805607105 (1 nodes), 808ns from start of poll, 173.784782ms from cluster start. Timeout at 2025-02-09 16:52:22.581255761 +0000 UTC m=+16.911488447, 105.881342ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.581Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.475374419 +0000 UTC m=+16.805607105 (1 nodes), 808ns from start of poll, 173.784782ms from cluster start. Timeout at 2025-02-09 16:52:22.581255761 +0000 UTC m=+16.911488447, 105.881342ms after stability -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering candidate state: node="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Candidate]" term=3 - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering candidate state: node="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Candidate]" term=3 -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.581Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.581Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering follower state: follower="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Follower]" leader= - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: entering follower state: follower="Node at 8ed6a849-50ac-04da-d74b-8e21f4edc48e [Follower]" leader= -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: aborting pipeline replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: aborting pipeline replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: election won: tally=2 - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: election won: tally=2 -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering leader state: leader="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Leader]" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: entering leader state: leader="Node at daf78732-292a-d894-97bb-bc3d80cb2891 [Leader]" -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: added peer, starting replication: peer=server-8ed6a849-50ac-04da-d74b-8e21f4edc48e - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: added peer, starting replication: peer=server-8ed6a849-50ac-04da-d74b-8e21f4edc48e -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: added peer, starting replication: peer=server-bde78719-276c-0151-d7d9-2f2d65f22669 - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: added peer, starting replication: peer=server-bde78719-276c-0151-d7d9-2f2d65f22669 -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: pipelining replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: pipelining replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" -testLoggerAdapter verbose: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: aborting pipeline replication: peer="{Voter server-daf78732-292a-d894-97bb-bc3d80cb2891 daf78732-292a-d894-97bb-bc3d80cb2891}" - testing.go:149: server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: 2025-02-09T16:52:22.581Z [INFO] server-8ed6a849-50ac-04da-d74b-8e21f4edc48e: aborting pipeline replication: peer="{Voter server-daf78732-292a-d894-97bb-bc3d80cb2891 daf78732-292a-d894-97bb-bc3d80cb2891}" -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: pipelining replication: peer="{Voter server-8ed6a849-50ac-04da-d74b-8e21f4edc48e 8ed6a849-50ac-04da-d74b-8e21f4edc48e}" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.581Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: pipelining replication: peer="{Voter server-8ed6a849-50ac-04da-d74b-8e21f4edc48e 8ed6a849-50ac-04da-d74b-8e21f4edc48e}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.687Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.581852666 +0000 UTC m=+16.912085354 (1 nodes), 330.798µs from start of poll, 280.263031ms from cluster start. Timeout at 2025-02-09 16:52:22.687060591 +0000 UTC m=+17.017293279, 105.207925ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.687Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.581852666 +0000 UTC m=+16.912085354 (1 nodes), 330.798µs from start of poll, 280.263031ms from cluster start. Timeout at 2025-02-09 16:52:22.687060591 +0000 UTC m=+17.017293279, 105.207925ms after stability -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.687Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: aborting pipeline replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.687Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: aborting pipeline replication: peer="{Voter server-bde78719-276c-0151-d7d9-2f2d65f22669 bde78719-276c-0151-d7d9-2f2d65f22669}" -testLoggerAdapter verbose: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.687Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: aborting pipeline replication: peer="{Voter server-8ed6a849-50ac-04da-d74b-8e21f4edc48e 8ed6a849-50ac-04da-d74b-8e21f4edc48e}" - testing.go:149: server-daf78732-292a-d894-97bb-bc3d80cb2891: 2025-02-09T16:52:22.687Z [INFO] server-daf78732-292a-d894-97bb-bc3d80cb2891: aborting pipeline replication: peer="{Voter server-8ed6a849-50ac-04da-d74b-8e21f4edc48e 8ed6a849-50ac-04da-d74b-8e21f4edc48e}" ---- PASS: TestRaft_LeadershipTransfer (0.39s) +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.225Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.225Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" +testLoggerAdapter verbose: server-e89fb56d-efec-5e6c-e0cb-da20735694f1: 2024-01-08T10:38:10.235Z [INFO] server-e89fb56d-efec-5e6c-e0cb-da20735694f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" + testing.go:149: server-e89fb56d-efec-5e6c-e0cb-da20735694f1: 2024-01-08T10:38:10.235Z [INFO] server-e89fb56d-efec-5e6c-e0cb-da20735694f1: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.236Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.236Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-53e42150-8fb1-53f1-4673-90760bc46e69 Address:53e42150-8fb1-53f1-4673-90760bc46e69} {Suffrage:Voter ID:server-e89fb56d-efec-5e6c-e0cb-da20735694f1 Address:e89fb56d-efec-5e6c-e0cb-da20735694f1} {Suffrage:Voter ID:server-57f024c7-3c00-7cad-990a-630c1a980232 Address:57f024c7-3c00-7cad-990a-630c1a980232}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.236Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.236Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.236Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering follower state: follower="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Follower]" leader= + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.236Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering follower state: follower="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Follower]" leader= +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.236Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering follower state: follower="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Follower]" leader= + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.236Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering follower state: follower="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Follower]" leader= +testLoggerAdapter verbose: server-e89fb56d-efec-5e6c-e0cb-da20735694f1: 2024-01-08T10:38:10.236Z [INFO] server-e89fb56d-efec-5e6c-e0cb-da20735694f1: entering follower state: follower="Node at e89fb56d-efec-5e6c-e0cb-da20735694f1 [Follower]" leader= + testing.go:149: server-e89fb56d-efec-5e6c-e0cb-da20735694f1: 2024-01-08T10:38:10.236Z [INFO] server-e89fb56d-efec-5e6c-e0cb-da20735694f1: entering follower state: follower="Node at e89fb56d-efec-5e6c-e0cb-da20735694f1 [Follower]" leader= +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.298Z [WARN] server-57f024c7-3c00-7cad-990a-630c1a980232: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.298Z [WARN] server-57f024c7-3c00-7cad-990a-630c1a980232: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.298Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering candidate state: node="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Candidate]" term=2 + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.298Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering candidate state: node="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Candidate]" term=2 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: election won: tally=2 + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: election won: tally=2 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering leader state: leader="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Leader]" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering leader state: leader="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Leader]" +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: added peer, starting replication: peer=server-53e42150-8fb1-53f1-4673-90760bc46e69 + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: added peer, starting replication: peer=server-53e42150-8fb1-53f1-4673-90760bc46e69 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: added peer, starting replication: peer=server-e89fb56d-efec-5e6c-e0cb-da20735694f1 + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.299Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: added peer, starting replication: peer=server-e89fb56d-efec-5e6c-e0cb-da20735694f1 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.300Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: pipelining replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.300Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: pipelining replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.301Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: pipelining replication: peer="{Voter server-53e42150-8fb1-53f1-4673-90760bc46e69 53e42150-8fb1-53f1-4673-90760bc46e69}" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.301Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: pipelining replication: peer="{Voter server-53e42150-8fb1-53f1-4673-90760bc46e69 53e42150-8fb1-53f1-4673-90760bc46e69}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.406Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.301373085 +0000 UTC m=+18.453167241 (1 nodes), 64.863093ms from start of poll, 76.509505ms from cluster start. Timeout at 2024-01-08 10:38:10.406387717 +0000 UTC m=+18.558181859, 105.014618ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.406Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.301373085 +0000 UTC m=+18.453167241 (1 nodes), 64.863093ms from start of poll, 76.509505ms from cluster start. Timeout at 2024-01-08 10:38:10.406387717 +0000 UTC m=+18.558181859, 105.014618ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.406Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.406Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.518Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.406989113 +0000 UTC m=+18.558783232 (1 nodes), 3.435µs from start of poll, 182.125496ms from cluster start. Timeout at 2024-01-08 10:38:10.518476561 +0000 UTC m=+18.670270703, 111.487471ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.518Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.406989113 +0000 UTC m=+18.558783232 (1 nodes), 3.435µs from start of poll, 182.125496ms from cluster start. Timeout at 2024-01-08 10:38:10.518476561 +0000 UTC m=+18.670270703, 111.487471ms after stability +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.519Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering candidate state: node="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Candidate]" term=3 + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.519Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering candidate state: node="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Candidate]" term=3 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.520Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering follower state: follower="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Follower]" leader= + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.520Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: entering follower state: follower="Node at 57f024c7-3c00-7cad-990a-630c1a980232 [Follower]" leader= +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.520Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: aborting pipeline replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.520Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: aborting pipeline replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.520Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.520Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: election won: tally=2 + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: election won: tally=2 +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering leader state: leader="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Leader]" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: entering leader state: leader="Node at 53e42150-8fb1-53f1-4673-90760bc46e69 [Leader]" +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: added peer, starting replication: peer=server-e89fb56d-efec-5e6c-e0cb-da20735694f1 + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.521Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: added peer, starting replication: peer=server-e89fb56d-efec-5e6c-e0cb-da20735694f1 +testLoggerAdapter verbose: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.522Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: aborting pipeline replication: peer="{Voter server-53e42150-8fb1-53f1-4673-90760bc46e69 53e42150-8fb1-53f1-4673-90760bc46e69}" + testing.go:149: server-57f024c7-3c00-7cad-990a-630c1a980232: 2024-01-08T10:38:10.522Z [INFO] server-57f024c7-3c00-7cad-990a-630c1a980232: aborting pipeline replication: peer="{Voter server-53e42150-8fb1-53f1-4673-90760bc46e69 53e42150-8fb1-53f1-4673-90760bc46e69}" +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.522Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: added peer, starting replication: peer=server-57f024c7-3c00-7cad-990a-630c1a980232 + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.522Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: added peer, starting replication: peer=server-57f024c7-3c00-7cad-990a-630c1a980232 +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.522Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: pipelining replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.522Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: pipelining replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.523Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: pipelining replication: peer="{Voter server-57f024c7-3c00-7cad-990a-630c1a980232 57f024c7-3c00-7cad-990a-630c1a980232}" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.523Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: pipelining replication: peer="{Voter server-57f024c7-3c00-7cad-990a-630c1a980232 57f024c7-3c00-7cad-990a-630c1a980232}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.630Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.522617412 +0000 UTC m=+18.674411582 (1 nodes), 1.787399ms from start of poll, 297.753846ms from cluster start. Timeout at 2024-01-08 10:38:10.630376511 +0000 UTC m=+18.782170657, 107.759075ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.630Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.522617412 +0000 UTC m=+18.674411582 (1 nodes), 1.787399ms from start of poll, 297.753846ms from cluster start. Timeout at 2024-01-08 10:38:10.630376511 +0000 UTC m=+18.782170657, 107.759075ms after stability +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.631Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: aborting pipeline replication: peer="{Voter server-57f024c7-3c00-7cad-990a-630c1a980232 57f024c7-3c00-7cad-990a-630c1a980232}" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.631Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: aborting pipeline replication: peer="{Voter server-57f024c7-3c00-7cad-990a-630c1a980232 57f024c7-3c00-7cad-990a-630c1a980232}" +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.631Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: aborting pipeline replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.631Z [INFO] server-53e42150-8fb1-53f1-4673-90760bc46e69: aborting pipeline replication: peer="{Voter server-e89fb56d-efec-5e6c-e0cb-da20735694f1 e89fb56d-efec-5e6c-e0cb-da20735694f1}" +testLoggerAdapter verbose: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.681Z [ERROR] server-53e42150-8fb1-53f1-4673-90760bc46e69: failed to heartbeat to: peer=e89fb56d-efec-5e6c-e0cb-da20735694f1 error="command timed out" + testing.go:149: server-53e42150-8fb1-53f1-4673-90760bc46e69: 2024-01-08T10:38:10.681Z [ERROR] server-53e42150-8fb1-53f1-4673-90760bc46e69: failed to heartbeat to: peer=e89fb56d-efec-5e6c-e0cb-da20735694f1 error="command timed out" +--- PASS: TestRaft_LeadershipTransfer (0.46s) === RUN TestRaft_LeadershipTransferWithOneNode -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.687Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cf0ad17-b26e-3439-e008-185bf80ab085 Address:0cf0ad17-b26e-3439-e008-185bf80ab085}]" - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.687Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0cf0ad17-b26e-3439-e008-185bf80ab085 Address:0cf0ad17-b26e-3439-e008-185bf80ab085}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.687Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.687Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.687Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering follower state: follower="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Follower]" leader= - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.687Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering follower state: follower="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Follower]" leader= -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [WARN] server-0cf0ad17-b26e-3439-e008-185bf80ab085: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [WARN] server-0cf0ad17-b26e-3439-e008-185bf80ab085: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering candidate state: node="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Candidate]" term=2 - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering candidate state: node="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Candidate]" term=2 -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: election won: tally=1 - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: election won: tally=1 -testLoggerAdapter verbose: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering leader state: leader="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Leader]" - testing.go:149: server-0cf0ad17-b26e-3439-e008-185bf80ab085: 2025-02-09T16:52:22.741Z [INFO] server-0cf0ad17-b26e-3439-e008-185bf80ab085: entering leader state: leader="Node at 0cf0ad17-b26e-3439-e008-185bf80ab085 [Leader]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.847Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.742049558 +0000 UTC m=+17.072282246 (1 nodes), 54.223405ms from start of poll, 54.384769ms from cluster start. Timeout at 2025-02-09 16:52:22.847520237 +0000 UTC m=+17.177752925, 105.470679ms after stability - testing.go:149: cluster: 2025-02-09T16:52:22.847Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.742049558 +0000 UTC m=+17.072282246 (1 nodes), 54.223405ms from start of poll, 54.384769ms from cluster start. Timeout at 2025-02-09 16:52:22.847520237 +0000 UTC m=+17.177752925, 105.470679ms after stability ---- PASS: TestRaft_LeadershipTransferWithOneNode (0.16s) +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.686Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8042552-6754-a1be-d878-18ef09730980 Address:f8042552-6754-a1be-d878-18ef09730980}]" + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.686Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f8042552-6754-a1be-d878-18ef09730980 Address:f8042552-6754-a1be-d878-18ef09730980}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.687Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.687Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.687Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering follower state: follower="Node at f8042552-6754-a1be-d878-18ef09730980 [Follower]" leader= + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.687Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering follower state: follower="Node at f8042552-6754-a1be-d878-18ef09730980 [Follower]" leader= +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.785Z [WARN] server-f8042552-6754-a1be-d878-18ef09730980: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.785Z [WARN] server-f8042552-6754-a1be-d878-18ef09730980: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering candidate state: node="Node at f8042552-6754-a1be-d878-18ef09730980 [Candidate]" term=2 + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering candidate state: node="Node at f8042552-6754-a1be-d878-18ef09730980 [Candidate]" term=2 +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: election won: tally=1 + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: election won: tally=1 +testLoggerAdapter verbose: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering leader state: leader="Node at f8042552-6754-a1be-d878-18ef09730980 [Leader]" + testing.go:149: server-f8042552-6754-a1be-d878-18ef09730980: 2024-01-08T10:38:10.786Z [INFO] server-f8042552-6754-a1be-d878-18ef09730980: entering leader state: leader="Node at f8042552-6754-a1be-d878-18ef09730980 [Leader]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.892Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.787091033 +0000 UTC m=+18.938885172 (1 nodes), 99.902193ms from start of poll, 101.896625ms from cluster start. Timeout at 2024-01-08 10:38:10.892329999 +0000 UTC m=+19.044124143, 105.238971ms after stability + testing.go:149: cluster: 2024-01-08T10:38:10.892Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.787091033 +0000 UTC m=+18.938885172 (1 nodes), 99.902193ms from start of poll, 101.896625ms from cluster start. Timeout at 2024-01-08 10:38:10.892329999 +0000 UTC m=+19.044124143, 105.238971ms after stability +--- PASS: TestRaft_LeadershipTransferWithOneNode (0.21s) === RUN TestRaft_LeadershipTransferWithSevenNodes -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:22.848Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:22.848Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: 2025-02-09T16:52:22.848Z [INFO] server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: 2025-02-09T16:52:22.848Z [INFO] server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-0f4025be-5174-0b44-72ba-b595d980b985: 2025-02-09T16:52:22.848Z [INFO] server-0f4025be-5174-0b44-72ba-b595d980b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-0f4025be-5174-0b44-72ba-b595d980b985: 2025-02-09T16:52:22.848Z [INFO] server-0f4025be-5174-0b44-72ba-b595d980b985: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.848Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.848Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: 2025-02-09T16:52:22.848Z [INFO] server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: 2025-02-09T16:52:22.848Z [INFO] server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-3cc26916-5f6b-8a84-b6b3-02666a516576: 2025-02-09T16:52:22.849Z [INFO] server-3cc26916-5f6b-8a84-b6b3-02666a516576: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-3cc26916-5f6b-8a84-b6b3-02666a516576: 2025-02-09T16:52:22.849Z [INFO] server-3cc26916-5f6b-8a84-b6b3-02666a516576: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: 2025-02-09T16:52:22.849Z [INFO] server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" - testing.go:149: server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: 2025-02-09T16:52:22.849Z [INFO] server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-ff3af693-2663-4c73-1120-f0d7708dccaf Address:ff3af693-2663-4c73-1120-f0d7708dccaf} {Suffrage:Voter ID:server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 Address:13a892b3-acfd-03c1-2fb0-cfbeb01431a0} {Suffrage:Voter ID:server-0f4025be-5174-0b44-72ba-b595d980b985 Address:0f4025be-5174-0b44-72ba-b595d980b985} {Suffrage:Voter ID:server-31dd4329-dc72-b900-0dfd-3207ef2a710c Address:31dd4329-dc72-b900-0dfd-3207ef2a710c} {Suffrage:Voter ID:server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed Address:ff3bb450-10c7-6ae2-ee02-746ea70dc9ed} {Suffrage:Voter ID:server-3cc26916-5f6b-8a84-b6b3-02666a516576 Address:3cc26916-5f6b-8a84-b6b3-02666a516576} {Suffrage:Voter ID:server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce Address:f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:22.849Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:22.849Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:22.850Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering follower state: follower="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Follower]" leader= - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:22.850Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering follower state: follower="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Follower]" leader= -testLoggerAdapter verbose: server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: 2025-02-09T16:52:22.850Z [INFO] server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: entering follower state: follower="Node at 13a892b3-acfd-03c1-2fb0-cfbeb01431a0 [Follower]" leader= - testing.go:149: server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: 2025-02-09T16:52:22.850Z [INFO] server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0: entering follower state: follower="Node at 13a892b3-acfd-03c1-2fb0-cfbeb01431a0 [Follower]" leader= -testLoggerAdapter verbose: server-0f4025be-5174-0b44-72ba-b595d980b985: 2025-02-09T16:52:22.850Z [INFO] server-0f4025be-5174-0b44-72ba-b595d980b985: entering follower state: follower="Node at 0f4025be-5174-0b44-72ba-b595d980b985 [Follower]" leader= - testing.go:149: server-0f4025be-5174-0b44-72ba-b595d980b985: 2025-02-09T16:52:22.850Z [INFO] server-0f4025be-5174-0b44-72ba-b595d980b985: entering follower state: follower="Node at 0f4025be-5174-0b44-72ba-b595d980b985 [Follower]" leader= -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.850Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering follower state: follower="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Follower]" leader= - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.850Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering follower state: follower="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Follower]" leader= -testLoggerAdapter verbose: server-3cc26916-5f6b-8a84-b6b3-02666a516576: 2025-02-09T16:52:22.851Z [INFO] server-3cc26916-5f6b-8a84-b6b3-02666a516576: entering follower state: follower="Node at 3cc26916-5f6b-8a84-b6b3-02666a516576 [Follower]" leader= - testing.go:149: server-3cc26916-5f6b-8a84-b6b3-02666a516576: 2025-02-09T16:52:22.851Z [INFO] server-3cc26916-5f6b-8a84-b6b3-02666a516576: entering follower state: follower="Node at 3cc26916-5f6b-8a84-b6b3-02666a516576 [Follower]" leader= -testLoggerAdapter verbose: server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: 2025-02-09T16:52:22.851Z [INFO] server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: entering follower state: follower="Node at ff3bb450-10c7-6ae2-ee02-746ea70dc9ed [Follower]" leader= - testing.go:149: server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: 2025-02-09T16:52:22.851Z [INFO] server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed: entering follower state: follower="Node at ff3bb450-10c7-6ae2-ee02-746ea70dc9ed [Follower]" leader= -testLoggerAdapter verbose: server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: 2025-02-09T16:52:22.851Z [INFO] server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: entering follower state: follower="Node at f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce [Follower]" leader= - testing.go:149: server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: 2025-02-09T16:52:22.851Z [INFO] server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce: entering follower state: follower="Node at f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce [Follower]" leader= -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.909Z [WARN] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.909Z [WARN] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering candidate state: node="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Candidate]" term=2 - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering candidate state: node="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Candidate]" term=2 -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: election won: tally=4 - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: election won: tally=4 -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering leader state: leader="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Leader]" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering leader state: leader="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Leader]" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-ff3af693-2663-4c73-1120-f0d7708dccaf - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-ff3af693-2663-4c73-1120-f0d7708dccaf -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-0f4025be-5174-0b44-72ba-b595d980b985 - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-0f4025be-5174-0b44-72ba-b595d980b985 -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-3cc26916-5f6b-8a84-b6b3-02666a516576 - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-3cc26916-5f6b-8a84-b6b3-02666a516576 -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: added peer, starting replication: peer=server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-ff3af693-2663-4c73-1120-f0d7708dccaf ff3af693-2663-4c73-1120-f0d7708dccaf}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-ff3af693-2663-4c73-1120-f0d7708dccaf ff3af693-2663-4c73-1120-f0d7708dccaf}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:22.910Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: pipelining replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.016Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.910512232 +0000 UTC m=+17.240744920 (1 nodes), 60.889999ms from start of poll, 62.248601ms from cluster start. Timeout at 2025-02-09 16:52:23.016222962 +0000 UTC m=+17.346455650, 105.71073ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.016Z [INFO] stable state for Leader reached at 2025-02-09 16:52:22.910512232 +0000 UTC m=+17.240744920 (1 nodes), 60.889999ms from start of poll, 62.248601ms from cluster start. Timeout at 2025-02-09 16:52:23.016222962 +0000 UTC m=+17.346455650, 105.71073ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.016Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:23.016Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.121Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.016360432 +0000 UTC m=+17.346593117 (6 nodes), 3.011µs from start of poll, 168.096798ms from cluster start. Timeout at 2025-02-09 16:52:23.121625638 +0000 UTC m=+17.451858326, 105.265209ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.121Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.016360432 +0000 UTC m=+17.346593117 (6 nodes), 3.011µs from start of poll, 168.096798ms from cluster start. Timeout at 2025-02-09 16:52:23.121625638 +0000 UTC m=+17.451858326, 105.265209ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.121Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:23.121Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.227Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.121762037 +0000 UTC m=+17.451994723 (1 nodes), 1.34µs from start of poll, 273.498404ms from cluster start. Timeout at 2025-02-09 16:52:23.227196324 +0000 UTC m=+17.557429012, 105.434289ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.227Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.121762037 +0000 UTC m=+17.451994723 (1 nodes), 1.34µs from start of poll, 273.498404ms from cluster start. Timeout at 2025-02-09 16:52:23.227196324 +0000 UTC m=+17.557429012, 105.434289ms after stability -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering candidate state: node="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Candidate]" term=3 - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering candidate state: node="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Candidate]" term=3 -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.227Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:23.227Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: election won: tally=4 - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: election won: tally=4 -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering leader state: leader="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Leader]" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: entering leader state: leader="Node at ff3af693-2663-4c73-1120-f0d7708dccaf [Leader]" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-0f4025be-5174-0b44-72ba-b595d980b985 - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-0f4025be-5174-0b44-72ba-b595d980b985 -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-31dd4329-dc72-b900-0dfd-3207ef2a710c - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-31dd4329-dc72-b900-0dfd-3207ef2a710c -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-3cc26916-5f6b-8a84-b6b3-02666a516576 - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-3cc26916-5f6b-8a84-b6b3-02666a516576 -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: added peer, starting replication: peer=server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.227Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering follower state: follower="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Follower]" leader= - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.227Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: entering follower state: follower="Node at 31dd4329-dc72-b900-0dfd-3207ef2a710c [Follower]" leader= -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.227Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.227Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-31dd4329-dc72-b900-0dfd-3207ef2a710c 31dd4329-dc72-b900-0dfd-3207ef2a710c}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-31dd4329-dc72-b900-0dfd-3207ef2a710c 31dd4329-dc72-b900-0dfd-3207ef2a710c}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.227Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.228Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.228Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: pipelining replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-ff3af693-2663-4c73-1120-f0d7708dccaf ff3af693-2663-4c73-1120-f0d7708dccaf}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-ff3af693-2663-4c73-1120-f0d7708dccaf ff3af693-2663-4c73-1120-f0d7708dccaf}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" -testLoggerAdapter verbose: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" - testing.go:149: server-31dd4329-dc72-b900-0dfd-3207ef2a710c: 2025-02-09T16:52:23.228Z [INFO] server-31dd4329-dc72-b900-0dfd-3207ef2a710c: aborting pipeline replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.333Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.227812891 +0000 UTC m=+17.558045579 (1 nodes), 393.432µs from start of poll, 379.54926ms from cluster start. Timeout at 2025-02-09 16:52:23.333380749 +0000 UTC m=+17.663613435, 105.567856ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.333Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.227812891 +0000 UTC m=+17.558045579 (1 nodes), 393.432µs from start of poll, 379.54926ms from cluster start. Timeout at 2025-02-09 16:52:23.333380749 +0000 UTC m=+17.663613435, 105.567856ms after stability -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-0f4025be-5174-0b44-72ba-b595d980b985 0f4025be-5174-0b44-72ba-b595d980b985}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-31dd4329-dc72-b900-0dfd-3207ef2a710c 31dd4329-dc72-b900-0dfd-3207ef2a710c}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-31dd4329-dc72-b900-0dfd-3207ef2a710c 31dd4329-dc72-b900-0dfd-3207ef2a710c}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-ff3bb450-10c7-6ae2-ee02-746ea70dc9ed ff3bb450-10c7-6ae2-ee02-746ea70dc9ed}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-3cc26916-5f6b-8a84-b6b3-02666a516576 3cc26916-5f6b-8a84-b6b3-02666a516576}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce f5ed09c6-27fe-8bf2-7995-d7aeed6f55ce}" -testLoggerAdapter verbose: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" - testing.go:149: server-ff3af693-2663-4c73-1120-f0d7708dccaf: 2025-02-09T16:52:23.333Z [INFO] server-ff3af693-2663-4c73-1120-f0d7708dccaf: aborting pipeline replication: peer="{Voter server-13a892b3-acfd-03c1-2fb0-cfbeb01431a0 13a892b3-acfd-03c1-2fb0-cfbeb01431a0}" ---- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.49s) +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:10.900Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:10.900Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:10.902Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering follower state: follower="Node at f0272111-5736-6156-bf82-8f394c205f0e [Follower]" leader= + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:10.902Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering follower state: follower="Node at f0272111-5736-6156-bf82-8f394c205f0e [Follower]" leader= +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.903Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.903Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-54ba4e12-b2e3-2056-c415-bac8d7baa320: 2024-01-08T10:38:10.904Z [INFO] server-54ba4e12-b2e3-2056-c415-bac8d7baa320: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-54ba4e12-b2e3-2056-c415-bac8d7baa320: 2024-01-08T10:38:10.904Z [INFO] server-54ba4e12-b2e3-2056-c415-bac8d7baa320: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.904Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering follower state: follower="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Follower]" leader= + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.904Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering follower state: follower="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Follower]" leader= +testLoggerAdapter verbose: server-54ba4e12-b2e3-2056-c415-bac8d7baa320: 2024-01-08T10:38:10.904Z [INFO] server-54ba4e12-b2e3-2056-c415-bac8d7baa320: entering follower state: follower="Node at 54ba4e12-b2e3-2056-c415-bac8d7baa320 [Follower]" leader= + testing.go:149: server-54ba4e12-b2e3-2056-c415-bac8d7baa320: 2024-01-08T10:38:10.904Z [INFO] server-54ba4e12-b2e3-2056-c415-bac8d7baa320: entering follower state: follower="Node at 54ba4e12-b2e3-2056-c415-bac8d7baa320 [Follower]" leader= +testLoggerAdapter verbose: server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: 2024-01-08T10:38:10.905Z [INFO] server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: 2024-01-08T10:38:10.905Z [INFO] server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-c9ffd001-affd-4795-8b0f-0b693a7c3307: 2024-01-08T10:38:10.906Z [INFO] server-c9ffd001-affd-4795-8b0f-0b693a7c3307: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-c9ffd001-affd-4795-8b0f-0b693a7c3307: 2024-01-08T10:38:10.906Z [INFO] server-c9ffd001-affd-4795-8b0f-0b693a7c3307: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-f1a4d6f2-352f-9423-653f-256aeaea35fc: 2024-01-08T10:38:10.907Z [INFO] server-f1a4d6f2-352f-9423-653f-256aeaea35fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" + testing.go:149: server-f1a4d6f2-352f-9423-653f-256aeaea35fc: 2024-01-08T10:38:10.907Z [INFO] server-f1a4d6f2-352f-9423-653f-256aeaea35fc: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: 2024-01-08T10:38:10.908Z [INFO] server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: server-c9ffd001-affd-4795-8b0f-0b693a7c3307: 2024-01-08T10:38:10.910Z [INFO] server-c9ffd001-affd-4795-8b0f-0b693a7c3307: entering follower state: follower="Node at c9ffd001-affd-4795-8b0f-0b693a7c3307 [Follower]" leader= +testLoggerAdapter verbose: server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: 2024-01-08T10:38:10.910Z [INFO] server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: entering follower state: follower="Node at 31d75051-ef55-e8ca-8cae-155f2ef7e5ec [Follower]" leader= +testLoggerAdapter verbose: server-f1a4d6f2-352f-9423-653f-256aeaea35fc: 2024-01-08T10:38:10.911Z [INFO] server-f1a4d6f2-352f-9423-653f-256aeaea35fc: entering follower state: follower="Node at f1a4d6f2-352f-9423-653f-256aeaea35fc [Follower]" leader= + testing.go:149: server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: 2024-01-08T10:38:10.908Z [INFO] server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f0272111-5736-6156-bf82-8f394c205f0e Address:f0272111-5736-6156-bf82-8f394c205f0e} {Suffrage:Voter ID:server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 Address:55bea5a6-dfbf-5309-cf6e-0aa3038778f3} {Suffrage:Voter ID:server-54ba4e12-b2e3-2056-c415-bac8d7baa320 Address:54ba4e12-b2e3-2056-c415-bac8d7baa320} {Suffrage:Voter ID:server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec Address:31d75051-ef55-e8ca-8cae-155f2ef7e5ec} {Suffrage:Voter ID:server-c9ffd001-affd-4795-8b0f-0b693a7c3307 Address:c9ffd001-affd-4795-8b0f-0b693a7c3307} {Suffrage:Voter ID:server-f1a4d6f2-352f-9423-653f-256aeaea35fc Address:f1a4d6f2-352f-9423-653f-256aeaea35fc} {Suffrage:Voter ID:server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab Address:be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:10.912Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:10.912Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-c9ffd001-affd-4795-8b0f-0b693a7c3307: 2024-01-08T10:38:10.910Z [INFO] server-c9ffd001-affd-4795-8b0f-0b693a7c3307: entering follower state: follower="Node at c9ffd001-affd-4795-8b0f-0b693a7c3307 [Follower]" leader= + testing.go:149: server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: 2024-01-08T10:38:10.910Z [INFO] server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec: entering follower state: follower="Node at 31d75051-ef55-e8ca-8cae-155f2ef7e5ec [Follower]" leader= + testing.go:149: server-f1a4d6f2-352f-9423-653f-256aeaea35fc: 2024-01-08T10:38:10.911Z [INFO] server-f1a4d6f2-352f-9423-653f-256aeaea35fc: entering follower state: follower="Node at f1a4d6f2-352f-9423-653f-256aeaea35fc [Follower]" leader= +testLoggerAdapter verbose: server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: 2024-01-08T10:38:10.913Z [INFO] server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: entering follower state: follower="Node at be18fb4a-25ac-1d45-92a6-24eb0b1d5bab [Follower]" leader= + testing.go:149: server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: 2024-01-08T10:38:10.913Z [INFO] server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab: entering follower state: follower="Node at be18fb4a-25ac-1d45-92a6-24eb0b1d5bab [Follower]" leader= +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.961Z [WARN] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.961Z [WARN] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.961Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering candidate state: node="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Candidate]" term=2 + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.961Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering candidate state: node="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Candidate]" term=2 +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.962Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: election won: tally=4 + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.962Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: election won: tally=4 +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering leader state: leader="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Leader]" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering leader state: leader="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Leader]" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-f0272111-5736-6156-bf82-8f394c205f0e + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-f0272111-5736-6156-bf82-8f394c205f0e +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-54ba4e12-b2e3-2056-c415-bac8d7baa320 + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-54ba4e12-b2e3-2056-c415-bac8d7baa320 +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-c9ffd001-affd-4795-8b0f-0b693a7c3307 + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-c9ffd001-affd-4795-8b0f-0b693a7c3307 +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-f1a4d6f2-352f-9423-653f-256aeaea35fc + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.963Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-f1a4d6f2-352f-9423-653f-256aeaea35fc +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: added peer, starting replication: peer=server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-f0272111-5736-6156-bf82-8f394c205f0e f0272111-5736-6156-bf82-8f394c205f0e}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-f0272111-5736-6156-bf82-8f394c205f0e f0272111-5736-6156-bf82-8f394c205f0e}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.965Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:10.964Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: pipelining replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.070Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.963442759 +0000 UTC m=+19.115236923 (1 nodes), 50.536716ms from start of poll, 63.481965ms from cluster start. Timeout at 2024-01-08 10:38:11.070192466 +0000 UTC m=+19.221986590, 106.749667ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.070Z [INFO] stable state for Leader reached at 2024-01-08 10:38:10.963442759 +0000 UTC m=+19.115236923 (1 nodes), 50.536716ms from start of poll, 63.481965ms from cluster start. Timeout at 2024-01-08 10:38:11.070192466 +0000 UTC m=+19.221986590, 106.749667ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.071Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:11.071Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.198Z [INFO] stable state for Follower reached at 2024-01-08 10:38:11.071325663 +0000 UTC m=+19.223119791 (6 nodes), 8.959µs from start of poll, 171.364833ms from cluster start. Timeout at 2024-01-08 10:38:11.198139439 +0000 UTC m=+19.349933588, 126.813797ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.198Z [INFO] stable state for Follower reached at 2024-01-08 10:38:11.071325663 +0000 UTC m=+19.223119791 (6 nodes), 8.959µs from start of poll, 171.364833ms from cluster start. Timeout at 2024-01-08 10:38:11.198139439 +0000 UTC m=+19.349933588, 126.813797ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.198Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:11.198Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.306Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.198944773 +0000 UTC m=+19.350738881 (1 nodes), 5.119µs from start of poll, 298.983923ms from cluster start. Timeout at 2024-01-08 10:38:11.306263114 +0000 UTC m=+19.458057264, 107.318383ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.306Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.198944773 +0000 UTC m=+19.350738881 (1 nodes), 5.119µs from start of poll, 298.983923ms from cluster start. Timeout at 2024-01-08 10:38:11.306263114 +0000 UTC m=+19.458057264, 107.318383ms after stability +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.307Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering candidate state: node="Node at f0272111-5736-6156-bf82-8f394c205f0e [Candidate]" term=3 + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.307Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering candidate state: node="Node at f0272111-5736-6156-bf82-8f394c205f0e [Candidate]" term=3 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.307Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:11.307Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.308Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering follower state: follower="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Follower]" leader= + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.308Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: entering follower state: follower="Node at 55bea5a6-dfbf-5309-cf6e-0aa3038778f3 [Follower]" leader= +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.308Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.308Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: election won: tally=4 + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: election won: tally=4 +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering leader state: leader="Node at f0272111-5736-6156-bf82-8f394c205f0e [Leader]" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: entering leader state: leader="Node at f0272111-5736-6156-bf82-8f394c205f0e [Leader]" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-f0272111-5736-6156-bf82-8f394c205f0e f0272111-5736-6156-bf82-8f394c205f0e}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-f0272111-5736-6156-bf82-8f394c205f0e f0272111-5736-6156-bf82-8f394c205f0e}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.309Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.311Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-54ba4e12-b2e3-2056-c415-bac8d7baa320 + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.311Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-54ba4e12-b2e3-2056-c415-bac8d7baa320 +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.311Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.311Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-c9ffd001-affd-4795-8b0f-0b693a7c3307 + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-c9ffd001-affd-4795-8b0f-0b693a7c3307 +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-f1a4d6f2-352f-9423-653f-256aeaea35fc + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-f1a4d6f2-352f-9423-653f-256aeaea35fc +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: added peer, starting replication: peer=server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" +testLoggerAdapter verbose: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" + testing.go:149: server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: 2024-01-08T10:38:11.310Z [INFO] server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3: aborting pipeline replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 55bea5a6-dfbf-5309-cf6e-0aa3038778f3}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 55bea5a6-dfbf-5309-cf6e-0aa3038778f3}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.312Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.313Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: pipelining replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.418Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.310855361 +0000 UTC m=+19.462649527 (1 nodes), 2.675904ms from start of poll, 410.894569ms from cluster start. Timeout at 2024-01-08 10:38:11.418615851 +0000 UTC m=+19.570410097, 107.76057ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.418Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.310855361 +0000 UTC m=+19.462649527 (1 nodes), 2.675904ms from start of poll, 410.894569ms from cluster start. Timeout at 2024-01-08 10:38:11.418615851 +0000 UTC m=+19.570410097, 107.76057ms after stability +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-f1a4d6f2-352f-9423-653f-256aeaea35fc f1a4d6f2-352f-9423-653f-256aeaea35fc}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-31d75051-ef55-e8ca-8cae-155f2ef7e5ec 31d75051-ef55-e8ca-8cae-155f2ef7e5ec}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-be18fb4a-25ac-1d45-92a6-24eb0b1d5bab be18fb4a-25ac-1d45-92a6-24eb0b1d5bab}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 55bea5a6-dfbf-5309-cf6e-0aa3038778f3}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-55bea5a6-dfbf-5309-cf6e-0aa3038778f3 55bea5a6-dfbf-5309-cf6e-0aa3038778f3}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-54ba4e12-b2e3-2056-c415-bac8d7baa320 54ba4e12-b2e3-2056-c415-bac8d7baa320}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.419Z [INFO] server-f0272111-5736-6156-bf82-8f394c205f0e: aborting pipeline replication: peer="{Voter server-c9ffd001-affd-4795-8b0f-0b693a7c3307 c9ffd001-affd-4795-8b0f-0b693a7c3307}" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.470Z [ERROR] server-f0272111-5736-6156-bf82-8f394c205f0e: failed to heartbeat to: peer=55bea5a6-dfbf-5309-cf6e-0aa3038778f3 error="command timed out" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.470Z [ERROR] server-f0272111-5736-6156-bf82-8f394c205f0e: failed to heartbeat to: peer=55bea5a6-dfbf-5309-cf6e-0aa3038778f3 error="command timed out" +testLoggerAdapter verbose: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.470Z [ERROR] server-f0272111-5736-6156-bf82-8f394c205f0e: failed to heartbeat to: peer=c9ffd001-affd-4795-8b0f-0b693a7c3307 error="command timed out" + testing.go:149: server-f0272111-5736-6156-bf82-8f394c205f0e: 2024-01-08T10:38:11.470Z [ERROR] server-f0272111-5736-6156-bf82-8f394c205f0e: failed to heartbeat to: peer=c9ffd001-affd-4795-8b0f-0b693a7c3307 error="command timed out" +--- PASS: TestRaft_LeadershipTransferWithSevenNodes (0.58s) === RUN TestRaft_LeadershipTransferToInvalidID -testLoggerAdapter verbose: server-36711481-d8d1-a742-bc90-9c6178f7c44a: 2025-02-09T16:52:23.334Z [INFO] server-36711481-d8d1-a742-bc90-9c6178f7c44a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" - testing.go:149: server-36711481-d8d1-a742-bc90-9c6178f7c44a: 2025-02-09T16:52:23.334Z [INFO] server-36711481-d8d1-a742-bc90-9c6178f7c44a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.334Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.334Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" -testLoggerAdapter verbose: server-452e476a-64bb-d833-1585-badf05e101de: 2025-02-09T16:52:23.334Z [INFO] server-452e476a-64bb-d833-1585-badf05e101de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" - testing.go:149: server-452e476a-64bb-d833-1585-badf05e101de: 2025-02-09T16:52:23.334Z [INFO] server-452e476a-64bb-d833-1585-badf05e101de: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-36711481-d8d1-a742-bc90-9c6178f7c44a Address:36711481-d8d1-a742-bc90-9c6178f7c44a} {Suffrage:Voter ID:server-4aaa030e-ab84-021a-1acc-7101959b390f Address:4aaa030e-ab84-021a-1acc-7101959b390f} {Suffrage:Voter ID:server-452e476a-64bb-d833-1585-badf05e101de Address:452e476a-64bb-d833-1585-badf05e101de}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.334Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:23.334Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-36711481-d8d1-a742-bc90-9c6178f7c44a: 2025-02-09T16:52:23.334Z [INFO] server-36711481-d8d1-a742-bc90-9c6178f7c44a: entering follower state: follower="Node at 36711481-d8d1-a742-bc90-9c6178f7c44a [Follower]" leader= - testing.go:149: server-36711481-d8d1-a742-bc90-9c6178f7c44a: 2025-02-09T16:52:23.334Z [INFO] server-36711481-d8d1-a742-bc90-9c6178f7c44a: entering follower state: follower="Node at 36711481-d8d1-a742-bc90-9c6178f7c44a [Follower]" leader= -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.334Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering follower state: follower="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Follower]" leader= - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.334Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering follower state: follower="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Follower]" leader= -testLoggerAdapter verbose: server-452e476a-64bb-d833-1585-badf05e101de: 2025-02-09T16:52:23.334Z [INFO] server-452e476a-64bb-d833-1585-badf05e101de: entering follower state: follower="Node at 452e476a-64bb-d833-1585-badf05e101de [Follower]" leader= - testing.go:149: server-452e476a-64bb-d833-1585-badf05e101de: 2025-02-09T16:52:23.334Z [INFO] server-452e476a-64bb-d833-1585-badf05e101de: entering follower state: follower="Node at 452e476a-64bb-d833-1585-badf05e101de [Follower]" leader= -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [WARN] server-4aaa030e-ab84-021a-1acc-7101959b390f: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [WARN] server-4aaa030e-ab84-021a-1acc-7101959b390f: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering candidate state: node="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Candidate]" term=2 - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering candidate state: node="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Candidate]" term=2 -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: election won: tally=2 - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: election won: tally=2 -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering leader state: leader="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Leader]" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: entering leader state: leader="Node at 4aaa030e-ab84-021a-1acc-7101959b390f [Leader]" -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: added peer, starting replication: peer=server-36711481-d8d1-a742-bc90-9c6178f7c44a - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.393Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: added peer, starting replication: peer=server-36711481-d8d1-a742-bc90-9c6178f7c44a -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: added peer, starting replication: peer=server-452e476a-64bb-d833-1585-badf05e101de - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: added peer, starting replication: peer=server-452e476a-64bb-d833-1585-badf05e101de -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: pipelining replication: peer="{Voter server-452e476a-64bb-d833-1585-badf05e101de 452e476a-64bb-d833-1585-badf05e101de}" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: pipelining replication: peer="{Voter server-452e476a-64bb-d833-1585-badf05e101de 452e476a-64bb-d833-1585-badf05e101de}" -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: pipelining replication: peer="{Voter server-36711481-d8d1-a742-bc90-9c6178f7c44a 36711481-d8d1-a742-bc90-9c6178f7c44a}" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.394Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: pipelining replication: peer="{Voter server-36711481-d8d1-a742-bc90-9c6178f7c44a 36711481-d8d1-a742-bc90-9c6178f7c44a}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.499Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.394226414 +0000 UTC m=+17.724459102 (1 nodes), 59.374544ms from start of poll, 59.707206ms from cluster start. Timeout at 2025-02-09 16:52:23.499229907 +0000 UTC m=+17.829462595, 105.003493ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.499Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.394226414 +0000 UTC m=+17.724459102 (1 nodes), 59.374544ms from start of poll, 59.707206ms from cluster start. Timeout at 2025-02-09 16:52:23.499229907 +0000 UTC m=+17.829462595, 105.003493ms after stability -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.499Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: aborting pipeline replication: peer="{Voter server-452e476a-64bb-d833-1585-badf05e101de 452e476a-64bb-d833-1585-badf05e101de}" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.499Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: aborting pipeline replication: peer="{Voter server-452e476a-64bb-d833-1585-badf05e101de 452e476a-64bb-d833-1585-badf05e101de}" -testLoggerAdapter verbose: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.499Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: aborting pipeline replication: peer="{Voter server-36711481-d8d1-a742-bc90-9c6178f7c44a 36711481-d8d1-a742-bc90-9c6178f7c44a}" - testing.go:149: server-4aaa030e-ab84-021a-1acc-7101959b390f: 2025-02-09T16:52:23.499Z [INFO] server-4aaa030e-ab84-021a-1acc-7101959b390f: aborting pipeline replication: peer="{Voter server-36711481-d8d1-a742-bc90-9c6178f7c44a 36711481-d8d1-a742-bc90-9c6178f7c44a}" ---- PASS: TestRaft_LeadershipTransferToInvalidID (0.17s) +testLoggerAdapter verbose: server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: 2024-01-08T10:38:11.479Z [INFO] server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" + testing.go:149: server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: 2024-01-08T10:38:11.479Z [INFO] server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.480Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" +testLoggerAdapter verbose: server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: 2024-01-08T10:38:11.480Z [INFO] server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: entering follower state: follower="Node at 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 [Follower]" leader= + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.480Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.482Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering follower state: follower="Node at f956539a-be76-e801-ae78-e24f373b4565 [Follower]" leader= + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.482Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering follower state: follower="Node at f956539a-be76-e801-ae78-e24f373b4565 [Follower]" leader= +testLoggerAdapter verbose: server-96c78c78-175b-1d06-3742-3441c7520de8: 2024-01-08T10:38:11.482Z [INFO] server-96c78c78-175b-1d06-3742-3441c7520de8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" + testing.go:149: server-96c78c78-175b-1d06-3742-3441c7520de8: 2024-01-08T10:38:11.482Z [INFO] server-96c78c78-175b-1d06-3742-3441c7520de8: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 Address:00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6} {Suffrage:Voter ID:server-f956539a-be76-e801-ae78-e24f373b4565 Address:f956539a-be76-e801-ae78-e24f373b4565} {Suffrage:Voter ID:server-96c78c78-175b-1d06-3742-3441c7520de8 Address:96c78c78-175b-1d06-3742-3441c7520de8}]" + testing.go:149: server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: 2024-01-08T10:38:11.480Z [INFO] server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6: entering follower state: follower="Node at 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.483Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:11.483Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-96c78c78-175b-1d06-3742-3441c7520de8: 2024-01-08T10:38:11.483Z [INFO] server-96c78c78-175b-1d06-3742-3441c7520de8: entering follower state: follower="Node at 96c78c78-175b-1d06-3742-3441c7520de8 [Follower]" leader= + testing.go:149: server-96c78c78-175b-1d06-3742-3441c7520de8: 2024-01-08T10:38:11.483Z [INFO] server-96c78c78-175b-1d06-3742-3441c7520de8: entering follower state: follower="Node at 96c78c78-175b-1d06-3742-3441c7520de8 [Follower]" leader= +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [WARN] server-f956539a-be76-e801-ae78-e24f373b4565: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [WARN] server-f956539a-be76-e801-ae78-e24f373b4565: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering candidate state: node="Node at f956539a-be76-e801-ae78-e24f373b4565 [Candidate]" term=2 + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering candidate state: node="Node at f956539a-be76-e801-ae78-e24f373b4565 [Candidate]" term=2 +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: election won: tally=2 + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.566Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: election won: tally=2 +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering leader state: leader="Node at f956539a-be76-e801-ae78-e24f373b4565 [Leader]" + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: entering leader state: leader="Node at f956539a-be76-e801-ae78-e24f373b4565 [Leader]" +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: added peer, starting replication: peer=server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: added peer, starting replication: peer=server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: added peer, starting replication: peer=server-96c78c78-175b-1d06-3742-3441c7520de8 + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: added peer, starting replication: peer=server-96c78c78-175b-1d06-3742-3441c7520de8 +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: pipelining replication: peer="{Voter server-96c78c78-175b-1d06-3742-3441c7520de8 96c78c78-175b-1d06-3742-3441c7520de8}" + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.567Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: pipelining replication: peer="{Voter server-96c78c78-175b-1d06-3742-3441c7520de8 96c78c78-175b-1d06-3742-3441c7520de8}" +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.568Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: pipelining replication: peer="{Voter server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6}" + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.568Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: pipelining replication: peer="{Voter server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.674Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.569226533 +0000 UTC m=+19.721020684 (1 nodes), 86.052107ms from start of poll, 90.424886ms from cluster start. Timeout at 2024-01-08 10:38:11.674400533 +0000 UTC m=+19.826194650, 105.173966ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.674Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.569226533 +0000 UTC m=+19.721020684 (1 nodes), 86.052107ms from start of poll, 90.424886ms from cluster start. Timeout at 2024-01-08 10:38:11.674400533 +0000 UTC m=+19.826194650, 105.173966ms after stability +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.675Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: aborting pipeline replication: peer="{Voter server-96c78c78-175b-1d06-3742-3441c7520de8 96c78c78-175b-1d06-3742-3441c7520de8}" + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.675Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: aborting pipeline replication: peer="{Voter server-96c78c78-175b-1d06-3742-3441c7520de8 96c78c78-175b-1d06-3742-3441c7520de8}" +testLoggerAdapter verbose: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.675Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: aborting pipeline replication: peer="{Voter server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6}" + testing.go:149: server-f956539a-be76-e801-ae78-e24f373b4565: 2024-01-08T10:38:11.675Z [INFO] server-f956539a-be76-e801-ae78-e24f373b4565: aborting pipeline replication: peer="{Voter server-00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6 00fcbb5e-bf4f-8c45-c2d0-7561ca3a5ea6}" +--- PASS: TestRaft_LeadershipTransferToInvalidID (0.20s) === RUN TestRaft_LeadershipTransferToInvalidAddress -testLoggerAdapter verbose: server-7b71f52c-6c43-55cd-99e7-25a5c158e819: 2025-02-09T16:52:23.500Z [INFO] server-7b71f52c-6c43-55cd-99e7-25a5c158e819: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" - testing.go:149: server-7b71f52c-6c43-55cd-99e7-25a5c158e819: 2025-02-09T16:52:23.500Z [INFO] server-7b71f52c-6c43-55cd-99e7-25a5c158e819: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" -testLoggerAdapter verbose: server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: 2025-02-09T16:52:23.500Z [INFO] server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" - testing.go:149: server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: 2025-02-09T16:52:23.500Z [INFO] server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.500Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.500Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-7b71f52c-6c43-55cd-99e7-25a5c158e819 Address:7b71f52c-6c43-55cd-99e7-25a5c158e819} {Suffrage:Voter ID:server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f Address:e87cd036-16ed-bbe5-c7b4-b2555fd66b3f} {Suffrage:Voter ID:server-65b2dac5-96e3-ea00-e0af-1b2edb929829 Address:65b2dac5-96e3-ea00-e0af-1b2edb929829}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.500Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:23.500Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-7b71f52c-6c43-55cd-99e7-25a5c158e819: 2025-02-09T16:52:23.500Z [INFO] server-7b71f52c-6c43-55cd-99e7-25a5c158e819: entering follower state: follower="Node at 7b71f52c-6c43-55cd-99e7-25a5c158e819 [Follower]" leader= - testing.go:149: server-7b71f52c-6c43-55cd-99e7-25a5c158e819: 2025-02-09T16:52:23.500Z [INFO] server-7b71f52c-6c43-55cd-99e7-25a5c158e819: entering follower state: follower="Node at 7b71f52c-6c43-55cd-99e7-25a5c158e819 [Follower]" leader= -testLoggerAdapter verbose: server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: 2025-02-09T16:52:23.500Z [INFO] server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: entering follower state: follower="Node at e87cd036-16ed-bbe5-c7b4-b2555fd66b3f [Follower]" leader= - testing.go:149: server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: 2025-02-09T16:52:23.500Z [INFO] server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f: entering follower state: follower="Node at e87cd036-16ed-bbe5-c7b4-b2555fd66b3f [Follower]" leader= -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.500Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering follower state: follower="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Follower]" leader= - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.500Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering follower state: follower="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Follower]" leader= -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.554Z [WARN] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.554Z [WARN] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.554Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering candidate state: node="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Candidate]" term=2 - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.554Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering candidate state: node="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Candidate]" term=2 -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: election won: tally=2 - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: election won: tally=2 -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering leader state: leader="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Leader]" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: entering leader state: leader="Node at 65b2dac5-96e3-ea00-e0af-1b2edb929829 [Leader]" -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: added peer, starting replication: peer=server-7b71f52c-6c43-55cd-99e7-25a5c158e819 - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: added peer, starting replication: peer=server-7b71f52c-6c43-55cd-99e7-25a5c158e819 -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: added peer, starting replication: peer=server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: added peer, starting replication: peer=server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: pipelining replication: peer="{Voter server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f e87cd036-16ed-bbe5-c7b4-b2555fd66b3f}" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: pipelining replication: peer="{Voter server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f e87cd036-16ed-bbe5-c7b4-b2555fd66b3f}" -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: pipelining replication: peer="{Voter server-7b71f52c-6c43-55cd-99e7-25a5c158e819 7b71f52c-6c43-55cd-99e7-25a5c158e819}" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.555Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: pipelining replication: peer="{Voter server-7b71f52c-6c43-55cd-99e7-25a5c158e819 7b71f52c-6c43-55cd-99e7-25a5c158e819}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.660Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.555310022 +0000 UTC m=+17.885542710 (2 nodes), 54.980246ms from start of poll, 55.317653ms from cluster start. Timeout at 2025-02-09 16:52:23.66069976 +0000 UTC m=+17.990932446, 105.389736ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.660Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.555310022 +0000 UTC m=+17.885542710 (2 nodes), 54.980246ms from start of poll, 55.317653ms from cluster start. Timeout at 2025-02-09 16:52:23.66069976 +0000 UTC m=+17.990932446, 105.389736ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.660Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:23.660Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.765Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.66083766 +0000 UTC m=+17.991070346 (1 nodes), 1.111µs from start of poll, 160.845289ms from cluster start. Timeout at 2025-02-09 16:52:23.765845539 +0000 UTC m=+18.096078227, 105.007881ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.765Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.66083766 +0000 UTC m=+17.991070346 (1 nodes), 1.111µs from start of poll, 160.845289ms from cluster start. Timeout at 2025-02-09 16:52:23.765845539 +0000 UTC m=+18.096078227, 105.007881ms after stability -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.766Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: aborting pipeline replication: peer="{Voter server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f e87cd036-16ed-bbe5-c7b4-b2555fd66b3f}" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.766Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: aborting pipeline replication: peer="{Voter server-e87cd036-16ed-bbe5-c7b4-b2555fd66b3f e87cd036-16ed-bbe5-c7b4-b2555fd66b3f}" -testLoggerAdapter verbose: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.766Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: aborting pipeline replication: peer="{Voter server-7b71f52c-6c43-55cd-99e7-25a5c158e819 7b71f52c-6c43-55cd-99e7-25a5c158e819}" - testing.go:149: server-65b2dac5-96e3-ea00-e0af-1b2edb929829: 2025-02-09T16:52:23.766Z [INFO] server-65b2dac5-96e3-ea00-e0af-1b2edb929829: aborting pipeline replication: peer="{Voter server-7b71f52c-6c43-55cd-99e7-25a5c158e819 7b71f52c-6c43-55cd-99e7-25a5c158e819}" ---- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.27s) +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.682Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.682Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" +testLoggerAdapter verbose: server-a8e9af00-a4dd-5523-1c27-53c24057e517: 2024-01-08T10:38:11.683Z [INFO] server-a8e9af00-a4dd-5523-1c27-53c24057e517: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" + testing.go:149: server-a8e9af00-a4dd-5523-1c27-53c24057e517: 2024-01-08T10:38:11.683Z [INFO] server-a8e9af00-a4dd-5523-1c27-53c24057e517: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.683Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering follower state: follower="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Follower]" leader= + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.683Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering follower state: follower="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Follower]" leader= +testLoggerAdapter verbose: server-6c85be01-ec95-4576-06e8-8de0e31d61e5: 2024-01-08T10:38:11.684Z [INFO] server-6c85be01-ec95-4576-06e8-8de0e31d61e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" + testing.go:149: server-6c85be01-ec95-4576-06e8-8de0e31d61e5: 2024-01-08T10:38:11.684Z [INFO] server-6c85be01-ec95-4576-06e8-8de0e31d61e5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-27fe5bd7-87b4-23d2-7d8e-17513423f730 Address:27fe5bd7-87b4-23d2-7d8e-17513423f730} {Suffrage:Voter ID:server-a8e9af00-a4dd-5523-1c27-53c24057e517 Address:a8e9af00-a4dd-5523-1c27-53c24057e517} {Suffrage:Voter ID:server-6c85be01-ec95-4576-06e8-8de0e31d61e5 Address:6c85be01-ec95-4576-06e8-8de0e31d61e5}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.684Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:11.684Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-6c85be01-ec95-4576-06e8-8de0e31d61e5: 2024-01-08T10:38:11.684Z [INFO] server-6c85be01-ec95-4576-06e8-8de0e31d61e5: entering follower state: follower="Node at 6c85be01-ec95-4576-06e8-8de0e31d61e5 [Follower]" leader= + testing.go:149: server-6c85be01-ec95-4576-06e8-8de0e31d61e5: 2024-01-08T10:38:11.684Z [INFO] server-6c85be01-ec95-4576-06e8-8de0e31d61e5: entering follower state: follower="Node at 6c85be01-ec95-4576-06e8-8de0e31d61e5 [Follower]" leader= +testLoggerAdapter verbose: server-a8e9af00-a4dd-5523-1c27-53c24057e517: 2024-01-08T10:38:11.684Z [INFO] server-a8e9af00-a4dd-5523-1c27-53c24057e517: entering follower state: follower="Node at a8e9af00-a4dd-5523-1c27-53c24057e517 [Follower]" leader= + testing.go:149: server-a8e9af00-a4dd-5523-1c27-53c24057e517: 2024-01-08T10:38:11.684Z [INFO] server-a8e9af00-a4dd-5523-1c27-53c24057e517: entering follower state: follower="Node at a8e9af00-a4dd-5523-1c27-53c24057e517 [Follower]" leader= +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [WARN] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [WARN] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering candidate state: node="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Candidate]" term=2 + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering candidate state: node="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Candidate]" term=2 +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: election won: tally=2 + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.754Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: election won: tally=2 +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering leader state: leader="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Leader]" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: entering leader state: leader="Node at 27fe5bd7-87b4-23d2-7d8e-17513423f730 [Leader]" +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: added peer, starting replication: peer=server-a8e9af00-a4dd-5523-1c27-53c24057e517 + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: added peer, starting replication: peer=server-a8e9af00-a4dd-5523-1c27-53c24057e517 +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: added peer, starting replication: peer=server-6c85be01-ec95-4576-06e8-8de0e31d61e5 + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.755Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: added peer, starting replication: peer=server-6c85be01-ec95-4576-06e8-8de0e31d61e5 +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.756Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: pipelining replication: peer="{Voter server-6c85be01-ec95-4576-06e8-8de0e31d61e5 6c85be01-ec95-4576-06e8-8de0e31d61e5}" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.756Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: pipelining replication: peer="{Voter server-6c85be01-ec95-4576-06e8-8de0e31d61e5 6c85be01-ec95-4576-06e8-8de0e31d61e5}" +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.756Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: pipelining replication: peer="{Voter server-a8e9af00-a4dd-5523-1c27-53c24057e517 a8e9af00-a4dd-5523-1c27-53c24057e517}" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.756Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: pipelining replication: peer="{Voter server-a8e9af00-a4dd-5523-1c27-53c24057e517 a8e9af00-a4dd-5523-1c27-53c24057e517}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.863Z [INFO] stable state for Follower reached at 2024-01-08 10:38:11.757268034 +0000 UTC m=+19.909062452 (2 nodes), 72.61644ms from start of poll, 76.144918ms from cluster start. Timeout at 2024-01-08 10:38:11.862794649 +0000 UTC m=+20.014588791, 105.526339ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.863Z [INFO] stable state for Follower reached at 2024-01-08 10:38:11.757268034 +0000 UTC m=+19.909062452 (2 nodes), 72.61644ms from start of poll, 76.144918ms from cluster start. Timeout at 2024-01-08 10:38:11.862794649 +0000 UTC m=+20.014588791, 105.526339ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.863Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:11.863Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.969Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.863714456 +0000 UTC m=+20.015508577 (1 nodes), 4.067µs from start of poll, 182.591043ms from cluster start. Timeout at 2024-01-08 10:38:11.969168259 +0000 UTC m=+20.120962377, 105.4538ms after stability + testing.go:149: cluster: 2024-01-08T10:38:11.969Z [INFO] stable state for Leader reached at 2024-01-08 10:38:11.863714456 +0000 UTC m=+20.015508577 (1 nodes), 4.067µs from start of poll, 182.591043ms from cluster start. Timeout at 2024-01-08 10:38:11.969168259 +0000 UTC m=+20.120962377, 105.4538ms after stability +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.970Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: aborting pipeline replication: peer="{Voter server-6c85be01-ec95-4576-06e8-8de0e31d61e5 6c85be01-ec95-4576-06e8-8de0e31d61e5}" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.970Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: aborting pipeline replication: peer="{Voter server-6c85be01-ec95-4576-06e8-8de0e31d61e5 6c85be01-ec95-4576-06e8-8de0e31d61e5}" +testLoggerAdapter verbose: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.970Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: aborting pipeline replication: peer="{Voter server-a8e9af00-a4dd-5523-1c27-53c24057e517 a8e9af00-a4dd-5523-1c27-53c24057e517}" + testing.go:149: server-27fe5bd7-87b4-23d2-7d8e-17513423f730: 2024-01-08T10:38:11.970Z [INFO] server-27fe5bd7-87b4-23d2-7d8e-17513423f730: aborting pipeline replication: peer="{Voter server-a8e9af00-a4dd-5523-1c27-53c24057e517 a8e9af00-a4dd-5523-1c27-53c24057e517}" +--- PASS: TestRaft_LeadershipTransferToInvalidAddress (0.29s) === RUN TestRaft_LeadershipTransferToBehindServer -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:23.766Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:23.766Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:23.767Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering follower state: follower="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Follower]" leader= - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:23.767Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering follower state: follower="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Follower]" leader= -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.767Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.767Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" -testLoggerAdapter verbose: server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: 2025-02-09T16:52:23.767Z [INFO] server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" - testing.go:149: server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: 2025-02-09T16:52:23.767Z [INFO] server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-16e8b072-5145-b415-53d5-1de4318fabe9 Address:16e8b072-5145-b415-53d5-1de4318fabe9} {Suffrage:Voter ID:server-b1c3ed07-44db-be29-dee9-45cd87462e5c Address:b1c3ed07-44db-be29-dee9-45cd87462e5c} {Suffrage:Voter ID:server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 Address:9886f2fc-315e-8a1a-ea34-63a0c4d99af6}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.767Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:23.767Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.767Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering follower state: follower="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Follower]" leader= - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.767Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering follower state: follower="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Follower]" leader= -testLoggerAdapter verbose: server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: 2025-02-09T16:52:23.767Z [INFO] server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: entering follower state: follower="Node at 9886f2fc-315e-8a1a-ea34-63a0c4d99af6 [Follower]" leader= - testing.go:149: server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: 2025-02-09T16:52:23.767Z [INFO] server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6: entering follower state: follower="Node at 9886f2fc-315e-8a1a-ea34-63a0c4d99af6 [Follower]" leader= -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [WARN] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [WARN] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering candidate state: node="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Candidate]" term=2 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering candidate state: node="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Candidate]" term=2 -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: election won: tally=2 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: election won: tally=2 -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering leader state: leader="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Leader]" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering leader state: leader="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Leader]" -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: added peer, starting replication: peer=server-16e8b072-5145-b415-53d5-1de4318fabe9 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: added peer, starting replication: peer=server-16e8b072-5145-b415-53d5-1de4318fabe9 -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: added peer, starting replication: peer=server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: added peer, starting replication: peer=server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: pipelining replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: pipelining replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: pipelining replication: peer="{Voter server-16e8b072-5145-b415-53d5-1de4318fabe9 16e8b072-5145-b415-53d5-1de4318fabe9}" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:23.839Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: pipelining replication: peer="{Voter server-16e8b072-5145-b415-53d5-1de4318fabe9 16e8b072-5145-b415-53d5-1de4318fabe9}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.945Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.839838404 +0000 UTC m=+18.170071096 (1 nodes), 72.499296ms from start of poll, 73.017335ms from cluster start. Timeout at 2025-02-09 16:52:23.945432827 +0000 UTC m=+18.275665513, 105.594417ms after stability - testing.go:149: cluster: 2025-02-09T16:52:23.945Z [INFO] stable state for Leader reached at 2025-02-09 16:52:23.839838404 +0000 UTC m=+18.170071096 (1 nodes), 72.499296ms from start of poll, 73.017335ms from cluster start. Timeout at 2025-02-09 16:52:23.945432827 +0000 UTC m=+18.275665513, 105.594417ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:23.945Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:23.945Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.050Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.945593691 +0000 UTC m=+18.275826378 (2 nodes), 1.057µs from start of poll, 178.772617ms from cluster start. Timeout at 2025-02-09 16:52:24.050762378 +0000 UTC m=+18.380995066, 105.168688ms after stability - testing.go:149: cluster: 2025-02-09T16:52:24.050Z [INFO] stable state for Follower reached at 2025-02-09 16:52:23.945593691 +0000 UTC m=+18.275826378 (2 nodes), 1.057µs from start of poll, 178.772617ms from cluster start. Timeout at 2025-02-09 16:52:24.050762378 +0000 UTC m=+18.380995066, 105.168688ms after stability -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.078Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering candidate state: node="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Candidate]" term=3 - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.078Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering candidate state: node="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Candidate]" term=3 -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: election won: tally=2 - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: election won: tally=2 -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering leader state: leader="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Leader]" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: entering leader state: leader="Node at 16e8b072-5145-b415-53d5-1de4318fabe9 [Leader]" -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: added peer, starting replication: peer=server-b1c3ed07-44db-be29-dee9-45cd87462e5c - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: added peer, starting replication: peer=server-b1c3ed07-44db-be29-dee9-45cd87462e5c -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering follower state: follower="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Follower]" leader=16e8b072-5145-b415-53d5-1de4318fabe9 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: entering follower state: follower="Node at b1c3ed07-44db-be29-dee9-45cd87462e5c [Follower]" leader=16e8b072-5145-b415-53d5-1de4318fabe9 -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: aborting pipeline replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: aborting pipeline replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.079Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:24.079Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: aborting pipeline replication: peer="{Voter server-16e8b072-5145-b415-53d5-1de4318fabe9 16e8b072-5145-b415-53d5-1de4318fabe9}" - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [INFO] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: aborting pipeline replication: peer="{Voter server-16e8b072-5145-b415-53d5-1de4318fabe9 16e8b072-5145-b415-53d5-1de4318fabe9}" -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: added peer, starting replication: peer=server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: added peer, starting replication: peer=server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: pipelining replication: peer="{Voter server-b1c3ed07-44db-be29-dee9-45cd87462e5c b1c3ed07-44db-be29-dee9-45cd87462e5c}" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: pipelining replication: peer="{Voter server-b1c3ed07-44db-be29-dee9-45cd87462e5c b1c3ed07-44db-be29-dee9-45cd87462e5c}" -testLoggerAdapter verbose: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [WARN] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: rejecting vote request since our last term is greater: candidate=16e8b072-5145-b415-53d5-1de4318fabe9 last-term=3 last-candidate-term=2 - testing.go:149: server-b1c3ed07-44db-be29-dee9-45cd87462e5c: 2025-02-09T16:52:24.079Z [WARN] server-b1c3ed07-44db-be29-dee9-45cd87462e5c: rejecting vote request since our last term is greater: candidate=16e8b072-5145-b415-53d5-1de4318fabe9 last-term=3 last-candidate-term=2 -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: pipelining replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.079Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: pipelining replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.185Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.079902814 +0000 UTC m=+18.410135503 (1 nodes), 383.14µs from start of poll, 313.081742ms from cluster start. Timeout at 2025-02-09 16:52:24.185240205 +0000 UTC m=+18.515472892, 105.337389ms after stability - testing.go:149: cluster: 2025-02-09T16:52:24.185Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.079902814 +0000 UTC m=+18.410135503 (1 nodes), 383.14µs from start of poll, 313.081742ms from cluster start. Timeout at 2025-02-09 16:52:24.185240205 +0000 UTC m=+18.515472892, 105.337389ms after stability -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.185Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: aborting pipeline replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.185Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: aborting pipeline replication: peer="{Voter server-9886f2fc-315e-8a1a-ea34-63a0c4d99af6 9886f2fc-315e-8a1a-ea34-63a0c4d99af6}" -testLoggerAdapter verbose: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.185Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: aborting pipeline replication: peer="{Voter server-b1c3ed07-44db-be29-dee9-45cd87462e5c b1c3ed07-44db-be29-dee9-45cd87462e5c}" - testing.go:149: server-16e8b072-5145-b415-53d5-1de4318fabe9: 2025-02-09T16:52:24.185Z [INFO] server-16e8b072-5145-b415-53d5-1de4318fabe9: aborting pipeline replication: peer="{Voter server-b1c3ed07-44db-be29-dee9-45cd87462e5c b1c3ed07-44db-be29-dee9-45cd87462e5c}" ---- PASS: TestRaft_LeadershipTransferToBehindServer (0.42s) +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:11.975Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:11.975Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:11.976Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering follower state: follower="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Follower]" leader= +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:11.976Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:11.976Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering follower state: follower="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Follower]" leader= + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:11.976Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" +testLoggerAdapter verbose: server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: 2024-01-08T10:38:11.977Z [INFO] server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" + testing.go:149: server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: 2024-01-08T10:38:11.977Z [INFO] server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-873f4247-d6cb-f3e4-0a73-ec02310c5843 Address:873f4247-d6cb-f3e4-0a73-ec02310c5843} {Suffrage:Voter ID:server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd Address:fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd} {Suffrage:Voter ID:server-03b7c3cb-533d-cd23-0206-c024e7a8a12d Address:03b7c3cb-533d-cd23-0206-c024e7a8a12d}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:11.978Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:11.978Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:11.978Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering follower state: follower="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Follower]" leader= + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:11.978Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering follower state: follower="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Follower]" leader= +testLoggerAdapter verbose: server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: 2024-01-08T10:38:11.978Z [INFO] server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: entering follower state: follower="Node at 03b7c3cb-533d-cd23-0206-c024e7a8a12d [Follower]" leader= + testing.go:149: server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: 2024-01-08T10:38:11.978Z [INFO] server-03b7c3cb-533d-cd23-0206-c024e7a8a12d: entering follower state: follower="Node at 03b7c3cb-533d-cd23-0206-c024e7a8a12d [Follower]" leader= +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.040Z [WARN] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.040Z [WARN] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.040Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering candidate state: node="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Candidate]" term=2 + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.040Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering candidate state: node="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Candidate]" term=2 +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: election won: tally=2 + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: election won: tally=2 +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering leader state: leader="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Leader]" + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering leader state: leader="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Leader]" +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: added peer, starting replication: peer=server-873f4247-d6cb-f3e4-0a73-ec02310c5843 + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: added peer, starting replication: peer=server-873f4247-d6cb-f3e4-0a73-ec02310c5843 +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: added peer, starting replication: peer=server-03b7c3cb-533d-cd23-0206-c024e7a8a12d + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.041Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: added peer, starting replication: peer=server-03b7c3cb-533d-cd23-0206-c024e7a8a12d +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.042Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: pipelining replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.042Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: pipelining replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.042Z [WARN] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: rejecting vote request since our last term is greater: candidate=fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.042Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: pipelining replication: peer="{Voter server-873f4247-d6cb-f3e4-0a73-ec02310c5843 873f4247-d6cb-f3e4-0a73-ec02310c5843}" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.042Z [WARN] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: rejecting vote request since our last term is greater: candidate=fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd last-term=2 last-candidate-term=1 + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.042Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: pipelining replication: peer="{Voter server-873f4247-d6cb-f3e4-0a73-ec02310c5843 873f4247-d6cb-f3e4-0a73-ec02310c5843}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.149Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.04332978 +0000 UTC m=+20.195123914 (1 nodes), 65.061486ms from start of poll, 68.555178ms from cluster start. Timeout at 2024-01-08 10:38:12.149476659 +0000 UTC m=+20.301270796, 106.146882ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.149Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.04332978 +0000 UTC m=+20.195123914 (1 nodes), 65.061486ms from start of poll, 68.555178ms from cluster start. Timeout at 2024-01-08 10:38:12.149476659 +0000 UTC m=+20.301270796, 106.146882ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.150Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:12.150Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.256Z [INFO] stable state for Follower reached at 2024-01-08 10:38:12.150282268 +0000 UTC m=+20.302076472 (2 nodes), 5.145µs from start of poll, 175.507736ms from cluster start. Timeout at 2024-01-08 10:38:12.256467944 +0000 UTC m=+20.408262073, 106.185601ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.256Z [INFO] stable state for Follower reached at 2024-01-08 10:38:12.150282268 +0000 UTC m=+20.302076472 (2 nodes), 5.145µs from start of poll, 175.507736ms from cluster start. Timeout at 2024-01-08 10:38:12.256467944 +0000 UTC m=+20.408262073, 106.185601ms after stability +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.414Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering candidate state: node="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Candidate]" term=3 + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.414Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering candidate state: node="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Candidate]" term=3 +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: election won: tally=2 + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: election won: tally=2 +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering leader state: leader="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Leader]" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: entering leader state: leader="Node at 873f4247-d6cb-f3e4-0a73-ec02310c5843 [Leader]" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: added peer, starting replication: peer=server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: added peer, starting replication: peer=server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: added peer, starting replication: peer=server-03b7c3cb-533d-cd23-0206-c024e7a8a12d + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.415Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: added peer, starting replication: peer=server-03b7c3cb-533d-cd23-0206-c024e7a8a12d +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.416Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:12.416Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.416Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering follower state: follower="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Follower]" leader= + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.416Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: entering follower state: follower="Node at fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd [Follower]" leader= +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.417Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: aborting pipeline replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.417Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: aborting pipeline replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.417Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: pipelining replication: peer="{Voter server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd}" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.417Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: pipelining replication: peer="{Voter server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd}" +testLoggerAdapter verbose: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.418Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: aborting pipeline replication: peer="{Voter server-873f4247-d6cb-f3e4-0a73-ec02310c5843 873f4247-d6cb-f3e4-0a73-ec02310c5843}" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.418Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: pipelining replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" + testing.go:149: server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: 2024-01-08T10:38:12.418Z [INFO] server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd: aborting pipeline replication: peer="{Voter server-873f4247-d6cb-f3e4-0a73-ec02310c5843 873f4247-d6cb-f3e4-0a73-ec02310c5843}" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.418Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: pipelining replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.523Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.418206019 +0000 UTC m=+20.570000189 (1 nodes), 1.628291ms from start of poll, 443.431453ms from cluster start. Timeout at 2024-01-08 10:38:12.523324675 +0000 UTC m=+20.675118825, 105.118636ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.523Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.418206019 +0000 UTC m=+20.570000189 (1 nodes), 1.628291ms from start of poll, 443.431453ms from cluster start. Timeout at 2024-01-08 10:38:12.523324675 +0000 UTC m=+20.675118825, 105.118636ms after stability +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.523Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: aborting pipeline replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.523Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: aborting pipeline replication: peer="{Voter server-03b7c3cb-533d-cd23-0206-c024e7a8a12d 03b7c3cb-533d-cd23-0206-c024e7a8a12d}" +testLoggerAdapter verbose: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.524Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: aborting pipeline replication: peer="{Voter server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd}" + testing.go:149: server-873f4247-d6cb-f3e4-0a73-ec02310c5843: 2024-01-08T10:38:12.524Z [INFO] server-873f4247-d6cb-f3e4-0a73-ec02310c5843: aborting pipeline replication: peer="{Voter server-fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd fd7d4256-6bf1-f7b7-d36e-c4130a3bf2dd}" +--- PASS: TestRaft_LeadershipTransferToBehindServer (0.55s) === RUN TestRaft_LeadershipTransferToItself -testLoggerAdapter verbose: server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: 2025-02-09T16:52:24.186Z [INFO] server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" - testing.go:149: server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: 2025-02-09T16:52:24.186Z [INFO] server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.186Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.186Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" -testLoggerAdapter verbose: server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: 2025-02-09T16:52:24.186Z [INFO] server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" - testing.go:149: server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: 2025-02-09T16:52:24.186Z [INFO] server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-498929cb-3dd0-d9bd-08bd-a53d623cd35f Address:498929cb-3dd0-d9bd-08bd-a53d623cd35f} {Suffrage:Voter ID:server-3fa5d42b-6b53-886b-8857-bb0eae8f716a Address:3fa5d42b-6b53-886b-8857-bb0eae8f716a} {Suffrage:Voter ID:server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b Address:74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.186Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:24.186Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: 2025-02-09T16:52:24.186Z [INFO] server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: entering follower state: follower="Node at 498929cb-3dd0-d9bd-08bd-a53d623cd35f [Follower]" leader= - testing.go:149: server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: 2025-02-09T16:52:24.186Z [INFO] server-498929cb-3dd0-d9bd-08bd-a53d623cd35f: entering follower state: follower="Node at 498929cb-3dd0-d9bd-08bd-a53d623cd35f [Follower]" leader= -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.186Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering follower state: follower="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Follower]" leader= - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.186Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering follower state: follower="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Follower]" leader= -testLoggerAdapter verbose: server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: 2025-02-09T16:52:24.187Z [INFO] server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: entering follower state: follower="Node at 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b [Follower]" leader= - testing.go:149: server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: 2025-02-09T16:52:24.187Z [INFO] server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b: entering follower state: follower="Node at 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b [Follower]" leader= -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [WARN] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [WARN] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering candidate state: node="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Candidate]" term=2 - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering candidate state: node="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Candidate]" term=2 -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: election won: tally=2 - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: election won: tally=2 -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering leader state: leader="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Leader]" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: entering leader state: leader="Node at 3fa5d42b-6b53-886b-8857-bb0eae8f716a [Leader]" -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: added peer, starting replication: peer=server-498929cb-3dd0-d9bd-08bd-a53d623cd35f - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: added peer, starting replication: peer=server-498929cb-3dd0-d9bd-08bd-a53d623cd35f -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: added peer, starting replication: peer=server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.272Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: added peer, starting replication: peer=server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.273Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: pipelining replication: peer="{Voter server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.273Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: pipelining replication: peer="{Voter server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}" -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.273Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: pipelining replication: peer="{Voter server-498929cb-3dd0-d9bd-08bd-a53d623cd35f 498929cb-3dd0-d9bd-08bd-a53d623cd35f}" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.273Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: pipelining replication: peer="{Voter server-498929cb-3dd0-d9bd-08bd-a53d623cd35f 498929cb-3dd0-d9bd-08bd-a53d623cd35f}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.378Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.273294041 +0000 UTC m=+18.603526729 (1 nodes), 86.423773ms from start of poll, 86.899185ms from cluster start. Timeout at 2025-02-09 16:52:24.37855957 +0000 UTC m=+18.708792257, 105.265528ms after stability - testing.go:149: cluster: 2025-02-09T16:52:24.378Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.273294041 +0000 UTC m=+18.603526729 (1 nodes), 86.423773ms from start of poll, 86.899185ms from cluster start. Timeout at 2025-02-09 16:52:24.37855957 +0000 UTC m=+18.708792257, 105.265528ms after stability -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: cannot transfer leadership to itself - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: cannot transfer leadership to itself -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: aborting pipeline replication: peer="{Voter server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: aborting pipeline replication: peer="{Voter server-74d7c61b-5e03-4b6b-9ea3-196e7c77f44b 74d7c61b-5e03-4b6b-9ea3-196e7c77f44b}" -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: aborting pipeline replication: peer="{Voter server-498929cb-3dd0-d9bd-08bd-a53d623cd35f 498929cb-3dd0-d9bd-08bd-a53d623cd35f}" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.378Z [INFO] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: aborting pipeline replication: peer="{Voter server-498929cb-3dd0-d9bd-08bd-a53d623cd35f 498929cb-3dd0-d9bd-08bd-a53d623cd35f}" -testLoggerAdapter verbose: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.429Z [ERROR] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: failed to heartbeat to: peer=498929cb-3dd0-d9bd-08bd-a53d623cd35f error="command timed out" - testing.go:149: server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: 2025-02-09T16:52:24.429Z [ERROR] server-3fa5d42b-6b53-886b-8857-bb0eae8f716a: failed to heartbeat to: peer=498929cb-3dd0-d9bd-08bd-a53d623cd35f error="command timed out" ---- PASS: TestRaft_LeadershipTransferToItself (0.24s) +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.529Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.529Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" +testLoggerAdapter verbose: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.530Z [INFO] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" + testing.go:149: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.530Z [INFO] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.531Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering follower state: follower="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Follower]" leader= +testLoggerAdapter verbose: server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: 2024-01-08T10:38:12.531Z [INFO] server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.531Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering follower state: follower="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Follower]" leader= + testing.go:149: server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: 2024-01-08T10:38:12.531Z [INFO] server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-10a609b5-7166-88a6-6479-02df57bea880 Address:10a609b5-7166-88a6-6479-02df57bea880} {Suffrage:Voter ID:server-b09eeb25-94f4-6665-df12-cb2bdb12b653 Address:b09eeb25-94f4-6665-df12-cb2bdb12b653} {Suffrage:Voter ID:server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 Address:3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}]" +testLoggerAdapter verbose: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.531Z [INFO] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: entering follower state: follower="Node at b09eeb25-94f4-6665-df12-cb2bdb12b653 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.532Z [INFO] starting stability test: raft-state=Leader + testing.go:149: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.531Z [INFO] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: entering follower state: follower="Node at b09eeb25-94f4-6665-df12-cb2bdb12b653 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T10:38:12.532Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: 2024-01-08T10:38:12.532Z [INFO] server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: entering follower state: follower="Node at 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 [Follower]" leader= + testing.go:149: server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: 2024-01-08T10:38:12.532Z [INFO] server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65: entering follower state: follower="Node at 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 [Follower]" leader= +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.590Z [WARN] server-10a609b5-7166-88a6-6479-02df57bea880: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.590Z [WARN] server-10a609b5-7166-88a6-6479-02df57bea880: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.590Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering candidate state: node="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Candidate]" term=2 + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.590Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering candidate state: node="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Candidate]" term=2 +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: election won: tally=2 + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: election won: tally=2 +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering leader state: leader="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Leader]" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: entering leader state: leader="Node at 10a609b5-7166-88a6-6479-02df57bea880 [Leader]" +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: added peer, starting replication: peer=server-b09eeb25-94f4-6665-df12-cb2bdb12b653 + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: added peer, starting replication: peer=server-b09eeb25-94f4-6665-df12-cb2bdb12b653 +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: added peer, starting replication: peer=server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.591Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: added peer, starting replication: peer=server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.592Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: pipelining replication: peer="{Voter server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.592Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: pipelining replication: peer="{Voter server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}" +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.592Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: pipelining replication: peer="{Voter server-b09eeb25-94f4-6665-df12-cb2bdb12b653 b09eeb25-94f4-6665-df12-cb2bdb12b653}" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.592Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: pipelining replication: peer="{Voter server-b09eeb25-94f4-6665-df12-cb2bdb12b653 b09eeb25-94f4-6665-df12-cb2bdb12b653}" +testLoggerAdapter verbose: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.593Z [WARN] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: rejecting vote request since our last term is greater: candidate=10a609b5-7166-88a6-6479-02df57bea880 last-term=2 last-candidate-term=1 + testing.go:149: server-b09eeb25-94f4-6665-df12-cb2bdb12b653: 2024-01-08T10:38:12.593Z [WARN] server-b09eeb25-94f4-6665-df12-cb2bdb12b653: rejecting vote request since our last term is greater: candidate=10a609b5-7166-88a6-6479-02df57bea880 last-term=2 last-candidate-term=1 +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.703Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.59349628 +0000 UTC m=+20.745290433 (1 nodes), 61.135766ms from start of poll, 64.835489ms from cluster start. Timeout at 2024-01-08 10:38:12.70279201 +0000 UTC m=+20.854586145, 109.295712ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.703Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.59349628 +0000 UTC m=+20.745290433 (1 nodes), 61.135766ms from start of poll, 64.835489ms from cluster start. Timeout at 2024-01-08 10:38:12.70279201 +0000 UTC m=+20.854586145, 109.295712ms after stability +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.703Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: cannot transfer leadership to itself + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.703Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: cannot transfer leadership to itself +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.703Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: aborting pipeline replication: peer="{Voter server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.703Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: aborting pipeline replication: peer="{Voter server-3d5aca8c-a51a-d2e5-21dc-1fdff2268e65 3d5aca8c-a51a-d2e5-21dc-1fdff2268e65}" +testLoggerAdapter verbose: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.704Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: aborting pipeline replication: peer="{Voter server-b09eeb25-94f4-6665-df12-cb2bdb12b653 b09eeb25-94f4-6665-df12-cb2bdb12b653}" + testing.go:149: server-10a609b5-7166-88a6-6479-02df57bea880: 2024-01-08T10:38:12.704Z [INFO] server-10a609b5-7166-88a6-6479-02df57bea880: aborting pipeline replication: peer="{Voter server-b09eeb25-94f4-6665-df12-cb2bdb12b653 b09eeb25-94f4-6665-df12-cb2bdb12b653}" +--- PASS: TestRaft_LeadershipTransferToItself (0.18s) === RUN TestRaft_LeadershipTransferLeaderRejectsClientRequests raft_test.go:2064: Disabled unreliable test --- SKIP: TestRaft_LeadershipTransferLeaderRejectsClientRequests (0.00s) === RUN TestRaft_LeadershipTransferLeaderReplicationTimeout -testLoggerAdapter verbose: server-920173a8-7fda-4235-7a39-d947808e55b5: 2025-02-09T16:52:24.430Z [INFO] server-920173a8-7fda-4235-7a39-d947808e55b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" - testing.go:149: server-920173a8-7fda-4235-7a39-d947808e55b5: 2025-02-09T16:52:24.430Z [INFO] server-920173a8-7fda-4235-7a39-d947808e55b5: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.430Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.430Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" -testLoggerAdapter verbose: server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: 2025-02-09T16:52:24.430Z [INFO] server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" - testing.go:149: server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: 2025-02-09T16:52:24.430Z [INFO] server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-920173a8-7fda-4235-7a39-d947808e55b5 Address:920173a8-7fda-4235-7a39-d947808e55b5} {Suffrage:Voter ID:server-96a2e38d-7567-c31d-918f-35cfd74c62aa Address:96a2e38d-7567-c31d-918f-35cfd74c62aa} {Suffrage:Voter ID:server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 Address:21c74c25-18fc-0c7b-81d3-e0432680eeb2}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.430Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:24.430Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: server-920173a8-7fda-4235-7a39-d947808e55b5: 2025-02-09T16:52:24.430Z [INFO] server-920173a8-7fda-4235-7a39-d947808e55b5: entering follower state: follower="Node at 920173a8-7fda-4235-7a39-d947808e55b5 [Follower]" leader= - testing.go:149: server-920173a8-7fda-4235-7a39-d947808e55b5: 2025-02-09T16:52:24.430Z [INFO] server-920173a8-7fda-4235-7a39-d947808e55b5: entering follower state: follower="Node at 920173a8-7fda-4235-7a39-d947808e55b5 [Follower]" leader= -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.430Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering follower state: follower="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Follower]" leader= - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.430Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering follower state: follower="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Follower]" leader= -testLoggerAdapter verbose: server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: 2025-02-09T16:52:24.430Z [INFO] server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: entering follower state: follower="Node at 21c74c25-18fc-0c7b-81d3-e0432680eeb2 [Follower]" leader= - testing.go:149: server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: 2025-02-09T16:52:24.430Z [INFO] server-21c74c25-18fc-0c7b-81d3-e0432680eeb2: entering follower state: follower="Node at 21c74c25-18fc-0c7b-81d3-e0432680eeb2 [Follower]" leader= -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [WARN] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [WARN] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering candidate state: node="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Candidate]" term=2 - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering candidate state: node="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Candidate]" term=2 -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: election won: tally=2 - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: election won: tally=2 -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering leader state: leader="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Leader]" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: entering leader state: leader="Node at 96a2e38d-7567-c31d-918f-35cfd74c62aa [Leader]" -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: added peer, starting replication: peer=server-920173a8-7fda-4235-7a39-d947808e55b5 - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: added peer, starting replication: peer=server-920173a8-7fda-4235-7a39-d947808e55b5 -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: added peer, starting replication: peer=server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: added peer, starting replication: peer=server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: pipelining replication: peer="{Voter server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 21c74c25-18fc-0c7b-81d3-e0432680eeb2}" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: pipelining replication: peer="{Voter server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 21c74c25-18fc-0c7b-81d3-e0432680eeb2}" -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: pipelining replication: peer="{Voter server-920173a8-7fda-4235-7a39-d947808e55b5 920173a8-7fda-4235-7a39-d947808e55b5}" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.518Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: pipelining replication: peer="{Voter server-920173a8-7fda-4235-7a39-d947808e55b5 920173a8-7fda-4235-7a39-d947808e55b5}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.624Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.518989253 +0000 UTC m=+18.849221941 (1 nodes), 88.140567ms from start of poll, 88.584761ms from cluster start. Timeout at 2025-02-09 16:52:24.624041826 +0000 UTC m=+18.954274517, 105.052576ms after stability - testing.go:149: cluster: 2025-02-09T16:52:24.624Z [INFO] stable state for Leader reached at 2025-02-09 16:52:24.518989253 +0000 UTC m=+18.849221941 (1 nodes), 88.140567ms from start of poll, 88.584761ms from cluster start. Timeout at 2025-02-09 16:52:24.624041826 +0000 UTC m=+18.954274517, 105.052576ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.624Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:24.624Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.729Z [INFO] stable state for Follower reached at 2025-02-09 16:52:24.62417429 +0000 UTC m=+18.954406975 (2 nodes), 1.006µs from start of poll, 193.769795ms from cluster start. Timeout at 2025-02-09 16:52:24.729542288 +0000 UTC m=+19.059774975, 105.368ms after stability - testing.go:149: cluster: 2025-02-09T16:52:24.729Z [INFO] stable state for Follower reached at 2025-02-09 16:52:24.62417429 +0000 UTC m=+18.954406975 (2 nodes), 1.006µs from start of poll, 193.769795ms from cluster start. Timeout at 2025-02-09 16:52:24.729542288 +0000 UTC m=+19.059774975, 105.368ms after stability -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.974Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: aborting pipeline replication: peer="{Voter server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 21c74c25-18fc-0c7b-81d3-e0432680eeb2}" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.974Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: aborting pipeline replication: peer="{Voter server-21c74c25-18fc-0c7b-81d3-e0432680eeb2 21c74c25-18fc-0c7b-81d3-e0432680eeb2}" -testLoggerAdapter verbose: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.974Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: aborting pipeline replication: peer="{Voter server-920173a8-7fda-4235-7a39-d947808e55b5 920173a8-7fda-4235-7a39-d947808e55b5}" - testing.go:149: server-96a2e38d-7567-c31d-918f-35cfd74c62aa: 2025-02-09T16:52:24.974Z [INFO] server-96a2e38d-7567-c31d-918f-35cfd74c62aa: aborting pipeline replication: peer="{Voter server-920173a8-7fda-4235-7a39-d947808e55b5 920173a8-7fda-4235-7a39-d947808e55b5}" ---- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (0.55s) +testLoggerAdapter verbose: server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: 2024-01-08T10:38:12.709Z [INFO] server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" + testing.go:149: server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: 2024-01-08T10:38:12.709Z [INFO] server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" +testLoggerAdapter verbose: server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: 2024-01-08T10:38:12.710Z [INFO] server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" + testing.go:149: server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: 2024-01-08T10:38:12.710Z [INFO] server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.711Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.711Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 Address:49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9} {Suffrage:Voter ID:server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb Address:d9504d62-ccbf-fe4c-2f67-9e000882c2cb} {Suffrage:Voter ID:server-51a050fc-9c87-fcda-6b33-0e022396b186 Address:51a050fc-9c87-fcda-6b33-0e022396b186}]" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.712Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:12.712Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: 2024-01-08T10:38:12.712Z [INFO] server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: entering follower state: follower="Node at d9504d62-ccbf-fe4c-2f67-9e000882c2cb [Follower]" leader= + testing.go:149: server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: 2024-01-08T10:38:12.712Z [INFO] server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb: entering follower state: follower="Node at d9504d62-ccbf-fe4c-2f67-9e000882c2cb [Follower]" leader= +testLoggerAdapter verbose: server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: 2024-01-08T10:38:12.712Z [INFO] server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: entering follower state: follower="Node at 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 [Follower]" leader= + testing.go:149: server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: 2024-01-08T10:38:12.712Z [INFO] server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9: entering follower state: follower="Node at 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 [Follower]" leader= +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.712Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering follower state: follower="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Follower]" leader= + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.712Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering follower state: follower="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Follower]" leader= +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.775Z [WARN] server-51a050fc-9c87-fcda-6b33-0e022396b186: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.775Z [WARN] server-51a050fc-9c87-fcda-6b33-0e022396b186: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering candidate state: node="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Candidate]" term=2 + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering candidate state: node="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Candidate]" term=2 +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: election won: tally=2 + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: election won: tally=2 +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering leader state: leader="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Leader]" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.776Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: entering leader state: leader="Node at 51a050fc-9c87-fcda-6b33-0e022396b186 [Leader]" +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: added peer, starting replication: peer=server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: added peer, starting replication: peer=server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: added peer, starting replication: peer=server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: added peer, starting replication: peer=server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: pipelining replication: peer="{Voter server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb d9504d62-ccbf-fe4c-2f67-9e000882c2cb}" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.777Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: pipelining replication: peer="{Voter server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb d9504d62-ccbf-fe4c-2f67-9e000882c2cb}" +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.778Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: pipelining replication: peer="{Voter server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9}" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:12.778Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: pipelining replication: peer="{Voter server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.886Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.778984673 +0000 UTC m=+20.930778819 (1 nodes), 66.805381ms from start of poll, 70.272808ms from cluster start. Timeout at 2024-01-08 10:38:12.88597709 +0000 UTC m=+21.037771418, 106.992599ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.886Z [INFO] stable state for Leader reached at 2024-01-08 10:38:12.778984673 +0000 UTC m=+20.930778819 (1 nodes), 66.805381ms from start of poll, 70.272808ms from cluster start. Timeout at 2024-01-08 10:38:12.88597709 +0000 UTC m=+21.037771418, 106.992599ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.886Z [INFO] starting stability test: raft-state=Follower + testing.go:149: cluster: 2024-01-08T10:38:12.886Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:12.992Z [INFO] stable state for Follower reached at 2024-01-08 10:38:12.886805747 +0000 UTC m=+21.038599857 (2 nodes), 5.7µs from start of poll, 178.093846ms from cluster start. Timeout at 2024-01-08 10:38:12.991935808 +0000 UTC m=+21.143729914, 105.130057ms after stability + testing.go:149: cluster: 2024-01-08T10:38:12.992Z [INFO] stable state for Follower reached at 2024-01-08 10:38:12.886805747 +0000 UTC m=+21.038599857 (2 nodes), 5.7µs from start of poll, 178.093846ms from cluster start. Timeout at 2024-01-08 10:38:12.991935808 +0000 UTC m=+21.143729914, 105.130057ms after stability +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:14.909Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: aborting pipeline replication: peer="{Voter server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb d9504d62-ccbf-fe4c-2f67-9e000882c2cb}" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:14.909Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: aborting pipeline replication: peer="{Voter server-d9504d62-ccbf-fe4c-2f67-9e000882c2cb d9504d62-ccbf-fe4c-2f67-9e000882c2cb}" +testLoggerAdapter verbose: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:14.910Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: aborting pipeline replication: peer="{Voter server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9}" + testing.go:149: server-51a050fc-9c87-fcda-6b33-0e022396b186: 2024-01-08T10:38:14.910Z [INFO] server-51a050fc-9c87-fcda-6b33-0e022396b186: aborting pipeline replication: peer="{Voter server-49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9 49446452-d9fb-5ac2-aed8-d4c0b8bbc8b9}" +--- PASS: TestRaft_LeadershipTransferLeaderReplicationTimeout (2.21s) === RUN TestRaft_LeadershipTransferStopRightAway --- PASS: TestRaft_LeadershipTransferStopRightAway (0.00s) === RUN TestRaft_GetConfigurationNoBootstrap -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:24.975Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:24.975Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" -testLoggerAdapter verbose: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:24.975Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" - testing.go:149: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:24.975Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:24.975Z [INFO] starting stability test: raft-state=Follower - testing.go:149: cluster: 2025-02-09T16:52:24.975Z [INFO] starting stability test: raft-state=Follower -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:24.975Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering follower state: follower="Node at e002409a-2eae-e863-14de-82de774e182d [Follower]" leader= - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:24.975Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering follower state: follower="Node at e002409a-2eae-e863-14de-82de774e182d [Follower]" leader= -testLoggerAdapter verbose: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:24.975Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: entering follower state: follower="Node at 24e230a7-ff12-1719-381d-ae2a73edb0d0 [Follower]" leader= - testing.go:149: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:24.975Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: entering follower state: follower="Node at 24e230a7-ff12-1719-381d-ae2a73edb0d0 [Follower]" leader= -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [WARN] server-e002409a-2eae-e863-14de-82de774e182d: heartbeat timeout reached, starting election: last-leader= - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [WARN] server-e002409a-2eae-e863-14de-82de774e182d: heartbeat timeout reached, starting election: last-leader= -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering candidate state: node="Node at e002409a-2eae-e863-14de-82de774e182d [Candidate]" term=2 - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering candidate state: node="Node at e002409a-2eae-e863-14de-82de774e182d [Candidate]" term=2 -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: election won: tally=2 - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: election won: tally=2 -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering leader state: leader="Node at e002409a-2eae-e863-14de-82de774e182d [Leader]" - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: entering leader state: leader="Node at e002409a-2eae-e863-14de-82de774e182d [Leader]" -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: added peer, starting replication: peer=server-24e230a7-ff12-1719-381d-ae2a73edb0d0 - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: added peer, starting replication: peer=server-24e230a7-ff12-1719-381d-ae2a73edb0d0 -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: pipelining replication: peer="{Voter server-24e230a7-ff12-1719-381d-ae2a73edb0d0 24e230a7-ff12-1719-381d-ae2a73edb0d0}" - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.043Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: pipelining replication: peer="{Voter server-24e230a7-ff12-1719-381d-ae2a73edb0d0 24e230a7-ff12-1719-381d-ae2a73edb0d0}" -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:25.149Z [INFO] stable state for Follower reached at 2025-02-09 16:52:25.043502237 +0000 UTC m=+19.373734925 (1 nodes), 67.713002ms from start of poll, 68.004739ms from cluster start. Timeout at 2025-02-09 16:52:25.149311304 +0000 UTC m=+19.479544002, 105.809077ms after stability - testing.go:149: cluster: 2025-02-09T16:52:25.149Z [INFO] stable state for Follower reached at 2025-02-09 16:52:25.043502237 +0000 UTC m=+19.373734925 (1 nodes), 67.713002ms from start of poll, 68.004739ms from cluster start. Timeout at 2025-02-09 16:52:25.149311304 +0000 UTC m=+19.479544002, 105.809077ms after stability -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:25.149Z [INFO] starting stability test: raft-state=Leader - testing.go:149: cluster: 2025-02-09T16:52:25.149Z [INFO] starting stability test: raft-state=Leader -testLoggerAdapter verbose: cluster: 2025-02-09T16:52:25.255Z [INFO] stable state for Leader reached at 2025-02-09 16:52:25.149452162 +0000 UTC m=+19.479684855 (1 nodes), 838ns from start of poll, 173.954669ms from cluster start. Timeout at 2025-02-09 16:52:25.25509387 +0000 UTC m=+19.585326556, 105.641701ms after stability - testing.go:149: cluster: 2025-02-09T16:52:25.255Z [INFO] stable state for Leader reached at 2025-02-09 16:52:25.149452162 +0000 UTC m=+19.479684855 (1 nodes), 838ns from start of poll, 173.954669ms from cluster start. Timeout at 2025-02-09 16:52:25.25509387 +0000 UTC m=+19.585326556, 105.641701ms after stability -testLoggerAdapter verbose: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:25.266Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" - testing.go:149: server-24e230a7-ff12-1719-381d-ae2a73edb0d0: 2025-02-09T16:52:25.266Z [INFO] server-24e230a7-ff12-1719-381d-ae2a73edb0d0: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e002409a-2eae-e863-14de-82de774e182d Address:e002409a-2eae-e863-14de-82de774e182d} {Suffrage:Voter ID:server-24e230a7-ff12-1719-381d-ae2a73edb0d0 Address:24e230a7-ff12-1719-381d-ae2a73edb0d0}]" -testLoggerAdapter verbose: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.266Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: aborting pipeline replication: peer="{Voter server-24e230a7-ff12-1719-381d-ae2a73edb0d0 24e230a7-ff12-1719-381d-ae2a73edb0d0}" - testing.go:149: server-e002409a-2eae-e863-14de-82de774e182d: 2025-02-09T16:52:25.266Z [INFO] server-e002409a-2eae-e863-14de-82de774e182d: aborting pipeline replication: peer="{Voter server-24e230a7-ff12-1719-381d-ae2a73edb0d0 24e230a7-ff12-1719-381d-ae2a73edb0d0}" ---- PASS: TestRaft_GetConfigurationNoBootstrap (0.29s) +testLoggerAdapter verbose: server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: 2024-01-08T10:38:14.915Z [INFO] server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" + testing.go:149: server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: 2024-01-08T10:38:14.915Z [INFO] server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:14.916Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:14.916Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" +testLoggerAdapter verbose: server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: 2024-01-08T10:38:14.916Z [INFO] server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: entering follower state: follower="Node at 68c7c05b-d179-a4ab-37b1-d2343ac1d873 [Follower]" leader= +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:14.916Z [INFO] starting stability test: raft-state=Follower + testing.go:149: server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: 2024-01-08T10:38:14.916Z [INFO] server-68c7c05b-d179-a4ab-37b1-d2343ac1d873: entering follower state: follower="Node at 68c7c05b-d179-a4ab-37b1-d2343ac1d873 [Follower]" leader= + testing.go:149: cluster: 2024-01-08T10:38:14.916Z [INFO] starting stability test: raft-state=Follower +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:14.917Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering follower state: follower="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Follower]" leader= + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:14.917Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering follower state: follower="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Follower]" leader= +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [WARN] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: heartbeat timeout reached, starting election: last-leader= + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [WARN] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: heartbeat timeout reached, starting election: last-leader= +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering candidate state: node="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Candidate]" term=2 + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering candidate state: node="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Candidate]" term=2 +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: election won: tally=2 + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.006Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: election won: tally=2 +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering leader state: leader="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Leader]" + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: entering leader state: leader="Node at ef0b4f92-bcfa-feaa-de41-4d1a43af682a [Leader]" +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: added peer, starting replication: peer=server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: added peer, starting replication: peer=server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: pipelining replication: peer="{Voter server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 68c7c05b-d179-a4ab-37b1-d2343ac1d873}" + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.007Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: pipelining replication: peer="{Voter server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 68c7c05b-d179-a4ab-37b1-d2343ac1d873}" +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:15.113Z [INFO] stable state for Follower reached at 2024-01-08 10:38:15.008246911 +0000 UTC m=+23.160041073 (1 nodes), 90.914482ms from start of poll, 93.358084ms from cluster start. Timeout at 2024-01-08 10:38:15.113798671 +0000 UTC m=+23.265592523, 105.55145ms after stability + testing.go:149: cluster: 2024-01-08T10:38:15.113Z [INFO] stable state for Follower reached at 2024-01-08 10:38:15.008246911 +0000 UTC m=+23.160041073 (1 nodes), 90.914482ms from start of poll, 93.358084ms from cluster start. Timeout at 2024-01-08 10:38:15.113798671 +0000 UTC m=+23.265592523, 105.55145ms after stability +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:15.113Z [INFO] starting stability test: raft-state=Leader + testing.go:149: cluster: 2024-01-08T10:38:15.113Z [INFO] starting stability test: raft-state=Leader +testLoggerAdapter verbose: cluster: 2024-01-08T10:38:15.219Z [INFO] stable state for Leader reached at 2024-01-08 10:38:15.113994939 +0000 UTC m=+23.265788788 (1 nodes), 2.365µs from start of poll, 199.105799ms from cluster start. Timeout at 2024-01-08 10:38:15.219767244 +0000 UTC m=+23.371561388, 105.7726ms after stability + testing.go:149: cluster: 2024-01-08T10:38:15.219Z [INFO] stable state for Leader reached at 2024-01-08 10:38:15.113994939 +0000 UTC m=+23.265788788 (1 nodes), 2.365µs from start of poll, 199.105799ms from cluster start. Timeout at 2024-01-08 10:38:15.219767244 +0000 UTC m=+23.371561388, 105.7726ms after stability +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.232Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.232Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 Address:68c7c05b-d179-a4ab-37b1-d2343ac1d873} {Suffrage:Voter ID:server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a Address:ef0b4f92-bcfa-feaa-de41-4d1a43af682a}]" +testLoggerAdapter verbose: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.233Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: aborting pipeline replication: peer="{Voter server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 68c7c05b-d179-a4ab-37b1-d2343ac1d873}" + testing.go:149: server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: 2024-01-08T10:38:15.233Z [INFO] server-ef0b4f92-bcfa-feaa-de41-4d1a43af682a: aborting pipeline replication: peer="{Voter server-68c7c05b-d179-a4ab-37b1-d2343ac1d873 68c7c05b-d179-a4ab-37b1-d2343ac1d873}" +--- PASS: TestRaft_GetConfigurationNoBootstrap (0.32s) === RUN TestTCPTransport_BadAddr --- PASS: TestTCPTransport_BadAddr (0.00s) === RUN TestTCPTransport_WithAdvertise @@ -3867,7 +3939,7 @@ === RUN TestRandomTimeout --- PASS: TestRandomTimeout (0.00s) === RUN TestNewSeed ---- PASS: TestNewSeed (0.00s) +--- PASS: TestNewSeed (0.03s) === RUN TestRandomTimeout_NoTime --- PASS: TestRandomTimeout_NoTime (0.00s) === RUN TestMin @@ -3875,11 +3947,11 @@ === RUN TestMax --- PASS: TestMax (0.00s) === RUN TestGenerateUUID ---- PASS: TestGenerateUUID (0.00s) +--- PASS: TestGenerateUUID (0.04s) === RUN TestBackoff --- PASS: TestBackoff (0.00s) PASS -ok github.com/hashicorp/raft 19.611s +ok github.com/hashicorp/raft 23.527s ? github.com/hashicorp/raft/bench [no test files] create-stamp debian/debhelper-build-stamp fakeroot debian/rules binary @@ -3911,12 +3983,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: including full source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/1225125/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/2584380 and its subdirectories -I: Current time: Sun Feb 9 04:52:31 -12 2025 -I: pbuilder-time-stamp: 1739119951 +I: removing directory /srv/workspace/pbuilder/1225125 and its subdirectories +I: Current time: Tue Jan 9 00:39:20 +14 2024 +I: pbuilder-time-stamp: 1704710360