Prune/Repack Uncompressed Stuck at 100%

I’ve had this happen a few times now, twice stuck at 100% and once seemingly stuck at 20%. Is this working as intended? There hasn’t been any CPU activity really for hours in this state.

If it helps, this is 0.16, running in GCP against a B2 backend.

Let me quote @MichaelEischer :point_up:

You could also send a SIGABRT signal to restic. Then the go runtime will abort and print a stacktrace of all running goroutines. That would help with debugging the hang.

It seems to be that if I do any more than about 200g at a time, this inevitably happens. Here’s the trace from the latest.

$ restic prune --repack-uncompressed --compression max

repository 99edf1bf opened (version 2, compression level max)
loading indexes...
loading all snapshots...
finding data that is still in use for 30 snapshots
[0:05] 100.00%  30 / 30 snapshots
searching used packs...
collecting packs for deletion and repacking
[0:57] 100.00%  306277 / 306277 packs processed

to repack:       1065961 blobs / 1.051 TiB
this removes:          0 blobs / 0 B
to delete:             0 blobs / 4.142 GiB
total prune:           0 blobs / 4.142 GiB
remaining:       2513774 blobs / 2.392 TiB
unused size after prune: 0 B (0.00% of remaining size)

deleting unreferenced packs
[0:09] 100.00%  252 / 252 files deleted
repacking packs
SIGABRT: abort.00%  222768 / 222768 packs repacked
PC=0x46a481 m=0 sigcode=0

goroutine 0 [idle]:
runtime.futex()
        /usr/local/go/src/runtime/sys_linux_amd64.s:554 +0x21 fp=0x7ffcaebb4af8 sp=0x7ffcaebb4af0 pc=0x46a481
runtime.futexsleep(0x10000000000?, 0x0?, 0x3?)
        /usr/local/go/src/runtime/os_linux.go:69 +0x36 fp=0x7ffcaebb4b48 sp=0x7ffcaebb4af8 pc=0x431596
runtime.notesleep(0x1c94248)
        /usr/local/go/src/runtime/lock_futex.go:160 +0x87 fp=0x7ffcaebb4b80 sp=0x7ffcaebb4b48 pc=0x40c107
runtime.mPark(...)
        /usr/local/go/src/runtime/proc.go:1531
runtime.stopm()
        /usr/local/go/src/runtime/proc.go:2335 +0x8c fp=0x7ffcaebb4bb0 sp=0x7ffcaebb4b80 pc=0x43bf8c
runtime.findRunnable()
        /usr/local/go/src/runtime/proc.go:3025 +0xabc fp=0x7ffcaebb4cb8 sp=0x7ffcaebb4bb0 pc=0x43d83c
runtime.schedule()
        /usr/local/go/src/runtime/proc.go:3378 +0xb1 fp=0x7ffcaebb4cf0 sp=0x7ffcaebb4cb8 pc=0x43e671
runtime.park_m(0xc00023bd40?)
        /usr/local/go/src/runtime/proc.go:3529 +0x12d fp=0x7ffcaebb4d20 sp=0x7ffcaebb4cf0 pc=0x43eb8d
runtime.mcall()
        /usr/local/go/src/runtime/asm_amd64.s:452 +0x43 fp=0x7ffcaebb4d30 sp=0x7ffcaebb4d20 pc=0x466463

goroutine 1 [semacquire, 962 minutes]:
runtime.gopark(0xc0006d2f20?, 0x440660?, 0xe0?, 0x16?, 0xc0000061a0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0006d2f18 sp=0xc0006d2ef8 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc010214350, 0x18?, 0x1, 0x0, 0x31?)
        /usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc0006d2f80 sp=0xc0006d2f18 pc=0x448e6f
sync.runtime_Semacquire(0xc01a043f50?)
        /usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc0006d2fb8 sp=0xc0006d2f80 pc=0x4643c7
sync.(*WaitGroup).Wait(0x121c200?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc0006d2fe0 sp=0xc0006d2fb8 pc=0x47474b
golang.org/x/sync/errgroup.(*Group).Wait(0xc010214340)
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x27 fp=0xc0006d3000 sp=0xc0006d2fe0 pc=
0x787ea7
github.com/restic/restic/internal/repository.Repack({0x1482d60, 0xc0001da0f0}, {0x148ce78?, 0xc000098750?}, {0x148ce78, 0
xc000098750?}, 0xc01c1d60c0, {0x1482290?, 0xc01c5f1bf0}, 0xc010092410)
        /restic/internal/repository/repack.go:44 +0x30f fp=0xc0006d30a8 sp=0xc0006d3000 pc=0x868e8f
main.doPrune({_, _}, {0x0, {0x0, 0x0}, 0x0, {0x12a107f, 0x2}, 0xc00011a5f0, {0x0, ...}, ...}, ...)
        /restic/cmd/restic/cmd_prune.go:733 +0x24d fp=0xc0006d3378 sp=0xc0006d30a8 pc=0xf6694d
main.runPruneWithRepo({_, _}, {0x0, {0x0, 0x0}, 0x0, {0x12a107f, 0x2}, 0xc00011a5f0, {0x0, ...}, ...}, ...)
        /restic/cmd/restic/cmd_prune.go:212 +0x40a fp=0xc0006d3828 sp=0xc0006d3378 pc=0xf62e6a
main.runPrune({_, _}, {0x0, {0x0, 0x0}, 0x0, {0x12a107f, 0x2}, 0xc00011a5f0, {0x0, ...}, ...}, ...)
        /restic/cmd/restic/cmd_prune.go:177 +0x513 fp=0xc0006d3a90 sp=0xc0006d3828 pc=0xf62953
main.glob..func19(0x1c7f840?, {0x12a20ac?, 0x3?, 0x3?})
        /restic/cmd/restic/cmd_prune.go:40 +0x9b fp=0xc0006d3c58 sp=0xc0006d3a90 pc=0xf61c7b
github.com/spf13/cobra.(*Command).execute(0x1c7f840, {0xc000119830, 0x3, 0x3})
        /home/build/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940 +0x862 fp=0xc0006d3d90 sp=0xc0006d3c58 pc=0x7
7a9a2
github.com/spf13/cobra.(*Command).ExecuteC(0x1c81500)
        /home/build/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068 +0x3bd fp=0xc0006d3e48 sp=0xc0006d3d90 pc=0x
77b21d
github.com/spf13/cobra.(*Command).Execute(...)
        /home/build/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992
github.com/spf13/cobra.(*Command).ExecuteContext(...)
        /home/build/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:985
main.main()
        /restic/cmd/restic/main.go:104 +0x1a5 fp=0xc0006d3f80 sp=0xc0006d3e48 pc=0xf821e5
runtime.main()
        /usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc0006d3fe0 sp=0xc0006d3f80 pc=0x437c87
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0006d3fe8 sp=0xc0006d3fe0 pc=0x468561

goroutine 2 [force gc (idle), 2 minutes]:
runtime.gopark(0x50ea93d788b2?, 0x0?, 0x0?, 0x0?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000060fb0 sp=0xc000060f90 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
        /usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc000060fe0 sp=0xc000060fb0 pc=0x437ef0
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000060fe8 sp=0xc000060fe0 pc=0x468561
created by runtime.init.6
        /usr/local/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1c92001?, 0x0?, 0x0?, 0x0?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000061780 sp=0xc000061760 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
        /usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0000617c8 sp=0xc000061780 pc=0x4241be
runtime.gcenable.func1()
        /usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000617e0 sp=0xc0000617c8 pc=0x419366
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000617e8 sp=0xc0000617e0 pc=0x468561
created by runtime.gcenable
        /usr/local/go/src/runtime/mgc.go:178 +0x6b

goroutine 4 [GC scavenge wait]:
runtime.gopark(0x50ea95d5a1a8?, 0xf9510e?, 0x0?, 0x0?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000061f70 sp=0xc000061f50 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x1c92f00)
        /usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc000061fa0 sp=0xc000061f70 pc=0x422093
runtime.bgscavenge(0x0?)
        /usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc000061fc8 sp=0xc000061fa0 pc=0x422685
runtime.gcenable.func2()
        /usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc000061fe0 sp=0xc000061fc8 pc=0x419306
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000061fe8 sp=0xc000061fe0 pc=0x468561
created by runtime.gcenable
        /usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 5 [finalizer wait]:
runtime.gopark(0x0?, 0xc01eaa4390?, 0x20?, 0x15?, 0x1000000010?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00013ae28 sp=0xc00013ae08 pc=0x4380b6
runtime.runfinq()
        /usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc00013afe0 sp=0xc00013ae28 pc=0x4183a7
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00013afe8 sp=0xc00013afe0 pc=0x468561
created by runtime.createfing
        /usr/local/go/src/runtime/mfinal.go:163 +0x45

goroutine 6 [chan receive, 963 minutes]:
runtime.gopark(0xc000062690?, 0xc0000626e8?, 0xd1?, 0xc0?, 0xc0000626e8?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0000626b8 sp=0xc000062698 pc=0x4380b6
runtime.chanrecv(0xc000114000, 0xc0000627a0, 0x1)
        /usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000062748 sp=0xc0000626b8 pc=0x406f9d
runtime.chanrecv2(0xc000114000?, 0xc0000627c0?)
        /usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000062770 sp=0xc000062748 pc=0x406ad8
github.com/restic/restic/internal/restic.init.0.func1.1()
        /restic/internal/restic/lock.go:361 +0x9d fp=0xc0000627e0 sp=0xc000062770 pc=0x7ca5dd
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000627e8 sp=0xc0000627e0 pc=0x468561
created by github.com/restic/restic/internal/restic.init.0.func1
        /restic/internal/restic/lock.go:358 +0x25

goroutine 17 [select, 963 minutes, locked to thread]:
runtime.gopark(0xc00005c7a8?, 0x2?, 0x32?, 0x84?, 0xc00005c7a4?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00005c618 sp=0xc00005c5f8 pc=0x4380b6
runtime.selectgo(0xc00005c7a8, 0xc00005c7a0, 0x0?, 0x0, 0x0?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00005c758 sp=0xc00005c618 pc=0x447d7e
runtime.ensureSigM.func1()
        /usr/local/go/src/runtime/signal_unix.go:1004 +0x1b0 fp=0xc00005c7e0 sp=0xc00005c758 pc=0x4603f0
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00005c7e8 sp=0xc00005c7e0 pc=0x468561
created by runtime.ensureSigM
        /usr/local/go/src/runtime/signal_unix.go:987 +0xbd

goroutine 33 [syscall, 963 minutes]:
runtime.notetsleepg(0x0?, 0x0?)
        /usr/local/go/src/runtime/lock_futex.go:236 +0x34 fp=0xc0001907a0 sp=0xc000190768 pc=0x40c414
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:152 +0x2f fp=0xc0001907c0 sp=0xc0001907a0 pc=0x464bef
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:23 +0x19 fp=0xc0001907e0 sp=0xc0001907c0 pc=0x4bc4b9
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001907e8 sp=0xc0001907e0 pc=0x468561
created by os/signal.Notify.func1.1
        /usr/local/go/src/os/signal/signal.go:151 +0x2a

goroutine 7 [select]:
runtime.gopark(0xc000062f88?, 0x3?, 0x3?, 0x0?, 0xc000062f72?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000062df8 sp=0xc000062dd8 pc=0x4380b6
runtime.selectgo(0xc000062f88, 0xc000062f6c, 0xc0002c0d00?, 0x0, 0x0?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc000062f38 sp=0xc000062df8 pc=0x447d7e
go.opencensus.io/stats/view.(*worker).start(0xc0002c0d00)
        /home/build/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0xad fp=0xc000062fc8 sp=0xc000062f38 pc
=0xe0ca2d
go.opencensus.io/stats/view.init.0.func1()
        /home/build/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x26 fp=0xc000062fe0 sp=0xc000062fc8 pc=
0xe0bc66
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000062fe8 sp=0xc000062fe0 pc=0x468561
created by go.opencensus.io/stats/view.init.0
        /home/build/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x96

goroutine 8 [GC worker (idle)]:
runtime.gopark(0x50ea946a8d22?, 0x3?, 0x72?, 0x65?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000063750 sp=0xc000063730 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0000637e0 sp=0xc000063750 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000637e8 sp=0xc0000637e0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 49 [GC worker (idle)]:
runtime.gopark(0x50ea946a8350?, 0x2?, 0xe8?, 0x98?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00018c750 sp=0xc00018c730 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00018c7e0 sp=0xc00018c750 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00018c7e8 sp=0xc00018c7e0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 50 [GC worker (idle), 10 minutes]:
runtime.gopark(0x505edcb58dd9?, 0x2?, 0x31?, 0xf0?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00018cf50 sp=0xc00018cf30 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00018cfe0 sp=0xc00018cf50 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00018cfe8 sp=0xc00018cfe0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 9 [GC worker (idle), 10 minutes]:
runtime.gopark(0x505edcb58c73?, 0x2?, 0x38?, 0x75?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000063f50 sp=0xc000063f30 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000063fe0 sp=0xc000063f50 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000063fe8 sp=0xc000063fe0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 51 [GC worker (idle), 12 minutes]:
runtime.gopark(0x5042eb170fdb?, 0x1?, 0x89?, 0x8b?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00018d750 sp=0xc00018d730 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc00018d7e0 sp=0xc00018d750 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00018d7e8 sp=0xc00018d7e0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 34 [GC worker (idle), 4 minutes]:
runtime.gopark(0x50cea250709b?, 0x2?, 0x3f?, 0xa5?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000190f50 sp=0xc000190f30 pc=0x4380b6
runtime.gcBgMarkWorker()
        /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc000190fe0 sp=0xc000190f50 pc=0x41b0d1
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000190fe8 sp=0xc000190fe0 pc=0x468561
created by runtime.gcBgMarkStartWorkers
        /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 35 [chan receive, 963 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000191680 sp=0xc000191660 pc=0x4380b6
runtime.chanrecv(0xc000194300, 0xc000191778, 0x1)
        /usr/local/go/src/runtime/chan.go:583 +0x49d fp=0xc000191710 sp=0xc000191680 pc=0x406f9d
runtime.chanrecv2(0x0?, 0x0?)
        /usr/local/go/src/runtime/chan.go:447 +0x18 fp=0xc000191738 sp=0xc000191710 pc=0x406ad8
main.CleanupHandler(0x0?)
        /restic/cmd/restic/cleanup.go:61 +0x47 fp=0xc0001917c8 sp=0xc000191738 pc=0xf40047
main.init.0.func1()
        /restic/cmd/restic/cleanup.go:21 +0x26 fp=0xc0001917e0 sp=0xc0001917c8 pc=0xf3fba6
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0001917e8 sp=0xc0001917e0 pc=0x468561
created by main.init.0
        /restic/cmd/restic/cleanup.go:21 +0x8b

goroutine 258 [semacquire, 26 minutes]:
runtime.gopark(0xc01248b0f8?, 0xc010092401?, 0xe0?, 0x39?, 0xc000f4ad30?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000f4ad30 sp=0xc000f4ad10 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc010214390, 0x67?, 0x1, 0x0, 0x40?)
        /usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc000f4ad98 sp=0xc000f4ad30 pc=0x448e6f
sync.runtime_Semacquire(0x4069f0?)
        /usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc000f4add0 sp=0xc000f4ad98 pc=0x4643c7
sync.(*WaitGroup).Wait(0xc01d5c28a0?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc000f4adf8 sp=0xc000f4add0 pc=0x47474b
golang.org/x/sync/errgroup.(*Group).Wait(0xc010214380)
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x27 fp=0xc000f4ae18 sp=0xc000f4adf8 pc=
0x787ea7
github.com/restic/restic/internal/repository.(*Repository).flushPacks(0xc000098750, {0x1482d60, 0xc0100924b0})
        /restic/internal/repository/repository.go:553 +0xa5 fp=0xc000f4ae40 sp=0xc000f4ae18 pc=0x86d9a5
github.com/restic/restic/internal/repository.(*Repository).Flush(0xc000098750, {0x1482d60, 0xc0100924b0})
        /restic/internal/repository/repository.go:511 +0x28 fp=0xc000f4ae78 sp=0xc000f4ae40 pc=0x86d4a8
github.com/restic/restic/internal/repository.repack({0x1482d60, 0xc0100924b0}, {0x148ce78?, 0xc000098750?}, {0x148ce78?,
0xc000098750?}, 0xc01c1d60c0, {0x1482290?, 0xc01c5f1bf0}, 0xc010092410)
        /restic/internal/repository/repack.go:135 +0x3b7 fp=0xc000f4af10 sp=0xc000f4ae78 pc=0x869417
github.com/restic/restic/internal/repository.Repack.func1()
        /restic/internal/repository/repack.go:40 +0x4e fp=0xc000f4af78 sp=0xc000f4af10 pc=0x86900e
golang.org/x/sync/errgroup.(*Group).Go.func1()
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x64 fp=0xc000f4afe0 sp=0xc000f4af78 pc=
0x788044
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000f4afe8 sp=0xc000f4afe0 pc=0x468561
created by golang.org/x/sync/errgroup.(*Group).Go
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0xa5

goroutine 251 [select]:
runtime.gopark(0xc00013bf78?, 0x3?, 0x5?, 0x0?, 0xc00013bf0a?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00013bd90 sp=0xc00013bd70 pc=0x4380b6
runtime.selectgo(0xc00013bf78, 0xc00013bf04, 0x1c92a00?, 0x0, 0x1415a8c738?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00013bed0 sp=0xc00013bd90 pc=0x447d7e
github.com/restic/restic/internal/ui/progress.(*Updater).run(0xc010214300)
        /restic/internal/ui/progress/updater.go:73 +0x186 fp=0xc00013bfc8 sp=0xc00013bed0 pc=0x7a6706
github.com/restic/restic/internal/ui/progress.NewUpdater.func1()
        /restic/internal/ui/progress/updater.go:40 +0x26 fp=0xc00013bfe0 sp=0xc00013bfc8 pc=0x7a6486
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00013bfe8 sp=0xc00013bfe0 pc=0x468561
created by github.com/restic/restic/internal/ui/progress.NewUpdater
        /restic/internal/ui/progress/updater.go:40 +0x16d

goroutine 257 [semacquire, 962 minutes]:
runtime.gopark(0x3?, 0x7?, 0x60?, 0x29?, 0xece8401dc1a1d53e?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc010a08678 sp=0xc010a08658 pc=0x4380b6
runtime.goparkunlock(...)
        /usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc010214390, 0x60?, 0x1, 0x0, 0xe5?)
        /usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc010a086e0 sp=0xc010a08678 pc=0x448e6f
sync.runtime_Semacquire(0x0?)
        /usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc010a08718 sp=0xc010a086e0 pc=0x4643c7
sync.(*WaitGroup).Wait(0x0?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc010a08740 sp=0xc010a08718 pc=0x47474b
golang.org/x/sync/errgroup.(*Group).Wait(0xc010214380)
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:53 +0x27 fp=0xc010a08760 sp=0xc010a08740 pc=
0x787ea7
github.com/restic/restic/internal/repository.(*Repository).StartPackUploader.func1()
        /restic/internal/repository/repository.go:534 +0x1d fp=0xc010a08778 sp=0xc010a08760 pc=0x86d8dd
golang.org/x/sync/errgroup.(*Group).Go.func1()
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x64 fp=0xc010a087e0 sp=0xc010a08778 pc=
0x788044
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc010a087e8 sp=0xc010a087e0 pc=0x468561
created by golang.org/x/sync/errgroup.(*Group).Go
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0xa5

goroutine 83 [select, 4 minutes]:
runtime.gopark(0xc014a2bf50?, 0x3?, 0x26?, 0x95?, 0xc014a2beaa?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc017c22cf8 sp=0xc017c22cd8 pc=0x4380b6
runtime.selectgo(0xc017c22f50, 0xc014a2bea4, 0xc000040090?, 0x0, 0x5?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc017c22e38 sp=0xc017c22cf8 pc=0x447d7e
main.refreshLocks({0x1482d60, 0xc0001da0f0}, {0x1489e58, 0xc0001180f0}, 0xc000082320, 0xc00008e120, 0xc00008e3c0)
        /restic/cmd/restic/lock.go:156 +0x1c5 fp=0xc017c22f98 sp=0xc017c22e38 pc=0xf80b45
main.lockRepository.func2()
        /restic/cmd/restic/lock.go:117 +0x3e fp=0xc017c22fe0 sp=0xc017c22f98 pc=0xf8093e
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc017c22fe8 sp=0xc017c22fe0 pc=0x468561
created by main.lockRepository
        /restic/cmd/restic/lock.go:117 +0x78a

goroutine 84 [select]:
runtime.gopark(0xc00005f750?, 0x4?, 0x4?, 0x0?, 0xc00005f690?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00005f4f8 sp=0xc00005f4d8 pc=0x4380b6
runtime.selectgo(0xc00005f750, 0xc00005f688, 0x0?, 0x0, 0x4?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc00005f638 sp=0xc00005f4f8 pc=0x447d7e
main.monitorLockRefresh({0x1482d60, 0xc0001da0f0}, 0xc000082320, 0xc00008e120, 0xc00008e3c0)
        /restic/cmd/restic/lock.go:220 +0x22c fp=0xc00005f7a8 sp=0xc00005f638 pc=0xf8124c
main.lockRepository.func3()
        /restic/cmd/restic/lock.go:118 +0x36 fp=0xc00005f7e0 sp=0xc00005f7a8 pc=0xf808d6
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00005f7e8 sp=0xc00005f7e0 pc=0x468561
created by main.lockRepository
        /restic/cmd/restic/lock.go:118 +0x82a

goroutine 252 [select, 562 minutes]:
runtime.gopark(0xc014a26750?, 0x6?, 0x10?, 0x24?, 0xc014a26564?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc014a263c8 sp=0xc014a263a8 pc=0x4380b6
runtime.selectgo(0xc014a26750, 0xc014a26558, 0x12b1a3c?, 0x0, 0xc014612540?, 0x1)
        /usr/local/go/src/runtime/select.go:327 +0x7be fp=0xc014a26508 sp=0xc014a263c8 pc=0x447d7e
net/http.(*persistConn).roundTrip(0xc0003da480, 0xc01d1421c0)
        /usr/local/go/src/net/http/transport.go:2638 +0x994 fp=0xc014a267c0 sp=0xc014a26508 pc=0x718fb4
net/http.(*Transport).roundTrip(0xc0001dc3c0, 0xc00f626d00)
        /usr/local/go/src/net/http/transport.go:603 +0x7fa fp=0xc014a269e8 sp=0xc014a267c0 pc=0x70ceda
net/http.(*Transport).RoundTrip(0x6?, 0xe0?)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x19 fp=0xc014a26a08 sp=0xc014a269e8 pc=0x6f4ff9
github.com/restic/restic/internal/backend/limiter.staticLimiter.roundTripper({0x0?, 0x0?}, {0x14785a0?, 0xc0001dc3c0?}, 0
xc00f626d00?)
        /restic/internal/backend/limiter/static_limiter.go:79 +0x1ea fp=0xc014a26aa0 sp=0xc014a26a08 pc=0x85d38a
github.com/restic/restic/internal/backend/limiter.staticLimiter.Transport.func1(0x45?)
        /restic/internal/backend/limiter/static_limiter.go:94 +0x2f fp=0xc014a26ad8 sp=0xc014a26aa0 pc=0x85d6af
github.com/restic/restic/internal/backend/limiter.roundTripper.RoundTrip(0x685ac5?, 0x0?)
        /restic/internal/backend/limiter/static_limiter.go:63 +0x1f fp=0xc014a26af0 sp=0xc014a26ad8 pc=0x85d15f
github.com/restic/restic/internal/backend/b2.(*sniffingRoundTripper).RoundTrip(0xc000119980, 0x12b20bd?)
        /restic/internal/backend/b2/b2.go:51 +0x2e fp=0xc014a26b28 sp=0xc014a26af0 pc=0xba1b6e
github.com/kurin/blazer/b2.(*clientTransport).RoundTrip(0xc0001e48e8, 0xc00f626d00)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/b2.go:159 +0xc2 fp=0xc014a
26c00 sp=0xc014a26b28 pc=0xb89562
github.com/kurin/blazer/base.makeNetRequest({0x1482d60, 0xc013e4f8b0}, 0xc00f626800, {0x1475a20, 0xc0001e48e8})
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/base/base.go:304 +0x12f fp=0x
c014a26cc8 sp=0xc014a26c00 pc=0xb6860f
github.com/kurin/blazer/base.(*b2Options).makeRequest(0xc0001d8800, {0x1482d60, 0xc013e4f8b0}, {0x12b0ced, 0xe}, {0x12a1d
50?, 0x2d4d?}, {0xc0006f2700?, 0x415bd0?}, {0x0, ...}, ...)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/base/base.go:408 +0x7ba fp=0x
c014a26f08 sp=0xc014a26cc8 pc=0xb691ba
github.com/kurin/blazer/base.(*URL).UploadFile(0xc00052d320, {0x1482d60, 0xc013e4f8b0}, {0x1475a40, 0xc013be8bd0}, 0x10db
8c6, {0xc012a44f60, 0x53}, {0x12bf4f3, 0x18}, ...)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/base/base.go:748 +0x4df fp=0x
c014a271a0 sp=0xc014a26f08 pc=0xb6beff
github.com/kurin/blazer/b2.(*b2URL).uploadFile(0x4f9a3e?, {0x1482d60?, 0xc013e4f8b0?}, {0x1475a40?, 0xc013be8bd0?}, 0xc01
46132a0?, {0xc012a44f60?, 0x4f965e?}, {0x12bf4f3, 0x18}, ...)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/baseline.go:405 +0x7b fp=0
xc014a27220 sp=0xc014a271a0 pc=0xb93a1b
github.com/kurin/blazer/b2.(*beURL).uploadFile.func1()
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/backend.go:511 +0x168 fp=0
xc014a27318 sp=0xc014a27220 pc=0xb8f768
github.com/kurin/blazer/b2.withBackoff({0x1482d60, 0xc013e4f8b0}, {0x1488d90, 0xc00011c1b0}, 0xc014a273d8)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/backend.go:752 +0x62 fp=0x
c014a273a0 sp=0xc014a27318 pc=0xb91222
github.com/kurin/blazer/b2.(*beURL).uploadFile(0x10db89e?, {0x1482d60?, 0xc013e4f8b0?}, {0x147bd28?, 0xc013be8bd0?}, 0xc0
146134e8?, {0xc012a44f60?, 0xc00021c280?}, {0x12bf4f3, 0x18}, ...)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/backend.go:522 +0x11d fp=0
xc014a27460 sp=0xc014a273a0 pc=0xb8f53d
github.com/kurin/blazer/b2.(*Writer).simpleWriteFile(0xc001e4e160)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/writer.go:307 +0x37a fp=0x
c014a275a0 sp=0xc014a27460 pc=0xb9d59a
github.com/kurin/blazer/b2.(*Writer).Close.func1()
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/writer.go:497 +0x1f3 fp=0x
c014a27610 sp=0xc014a275a0 pc=0xb9ea53
sync.(*Once).doSlow(0x40b6f6?, 0x10e2980?)
        /usr/local/go/src/sync/once.go:74 +0xc2 fp=0xc014a27670 sp=0xc014a27610 pc=0x472fe2
sync.(*Once).Do(...)
        /usr/local/go/src/sync/once.go:65
github.com/kurin/blazer/b2.(*Writer).Close(0xc001e4e160?)
        /home/build/go/pkg/mod/github.com/kurin/blazer@v0.5.4-0.20230113224640-3887e1ec64b5/b2/writer.go:483 +0x50 fp=0xc
014a276a0 sp=0xc014a27670 pc=0xb9e830
github.com/restic/restic/internal/backend/b2.(*b2Backend).Save(0xc0005224d0, {0x1482d60?, 0xc010092500?}, {0xa5?, 0x42?,
{0xc012dd2440?, 0x18?}}, {0x14830a8?, 0xc012d78570})
        /restic/internal/backend/b2/b2.go:236 +0x305 fp=0xc014a27778 sp=0xc014a276a0 pc=0xba3205
github.com/restic/restic/internal/backend/sema.(*connectionLimitedBackend).Save(0xc000622320, {0x1482d60, 0xc010092500},
{0xc0?, 0x37?, {0xc012dd2440?, 0x4b1a45?}}, {0x14830a8, 0xc012d78570})
        /restic/internal/backend/sema/backend.go:73 +0x1c6 fp=0xc014a27808 sp=0xc014a27778 pc=0xef8e86
github.com/restic/restic/internal/backend/logger.(*Backend).Save(0xc0001ca5b0, {0x1482d60, 0xc010092500}, {0x1, 0x1, {0xc
012dd2440, 0x40}}, {0x14830a8?, 0xc012d78570?})
        /restic/internal/backend/logger/log.go:31 +0x14f fp=0xc014a278b0 sp=0xc014a27808 pc=0xeee4ef
github.com/restic/restic/internal/backend/retry.(*Backend).Save.func1()
        /restic/internal/backend/retry/backend_retry.go:102 +0xa8 fp=0xc014a27968 sp=0xc014a278b0 pc=0x863f48
github.com/restic/restic/internal/backend/retry.retryNotifyErrorWithSuccess.func1()
        /restic/internal/backend/retry/backend_retry.go:47 +0x2f fp=0xc014a279a0 sp=0xc014a27968 pc=0x8638ef
github.com/cenkalti/backoff/v4.RetryNotifyWithTimer.func1()
        /home/build/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:18 +0x1b fp=0xc014a279b0 sp=0xc014a279a0 pc
=0x7e92fb
github.com/cenkalti/backoff/v4.doRetryNotify[...](0xc014a27ad8?, {0x7f8f1bf0b0d0, 0xc010182320}, 0xc014a27bd0, {0x0, 0x0?
})
        /home/build/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:88 +0x152 fp=0xc014a27aa0 sp=0xc014a279b0 p
c=0x7e97b2
github.com/cenkalti/backoff/v4.RetryNotifyWithTimer(0x7f8f1afa4828?, {0x7f8f1bf0b0d0?, 0xc010182320?}, 0xc000180c00?, {0x
0?, 0x0?})
        /home/build/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:61 +0x65 fp=0xc014a27af8 sp=0xc014a27aa0 pc
=0x7e9285
github.com/cenkalti/backoff/v4.RetryNotify(...)
        /home/build/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:49
github.com/restic/restic/internal/backend/retry.retryNotifyErrorWithSuccess(0x1264fe0?, {0x7f8f1bf0b0d0?, 0xc010182320?},
 0x4f65fc?, 0x0?)
        /restic/internal/backend/retry/backend_retry.go:55 +0x6b fp=0xc014a27b60 sp=0xc014a27af8 pc=0x86384b
github.com/restic/restic/internal/backend/retry.(*Backend).retry(0xc00061a3c0, {0x1482d60, 0xc010092500}, {0xc00f6201c8,
0x17}, 0xc01abf4000?)
        /restic/internal/backend/retry/backend_retry.go:77 +0x209 fp=0xc014a27c00 sp=0xc014a27b60 pc=0x863b49
github.com/restic/restic/internal/backend/retry.(*Backend).Save(0xc00061a3c0, {0x1482d60, 0xc010092500}, {0x1, 0x1, {0xc0
12dd2440, 0x40}}, {0x14830a8, 0xc012d78570})
        /restic/internal/backend/retry/backend_retry.go:96 +0x165 fp=0xc014a27c98 sp=0xc014a27c00 pc=0x863e25
github.com/restic/restic/internal/cache.(*Backend).Save(0xc0001180f0, {0x1482d60, 0xc010092500}, {0x1, 0x1, {0xc012dd2440
, 0x40}}, {0x14830a8?, 0xc012d78570?})
        /restic/internal/cache/backend.go:59 +0x2f2 fp=0xc014a27d40 sp=0xc014a27c98 pc=0x846932
github.com/restic/restic/internal/repository.(*Repository).savePacker(0xc000098750, {0x1482d60, 0xc010092500}, 0x1, 0xc01
0688000)
        /restic/internal/repository/packer_manager.go:169 +0x564 fp=0xc014a27ed0 sp=0xc014a27d40 pc=0x868384
github.com/restic/restic/internal/repository.newPackerUploader.func1()
        /restic/internal/repository/packer_uploader.go:37 +0xff fp=0xc014a27f78 sp=0xc014a27ed0 pc=0x86899f
golang.org/x/sync/errgroup.(*Group).Go.func1()
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:75 +0x64 fp=0xc014a27fe0 sp=0xc014a27f78 pc=
0x788044
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc014a27fe8 sp=0xc014a27fe0 pc=0x468561
created by golang.org/x/sync/errgroup.(*Group).Go
        /home/build/go/pkg/mod/golang.org/x/sync@v0.3.0/errgroup/errgroup.go:72 +0xa5

goroutine 8369 [IO wait, 562 minutes]:
runtime.gopark(0xe5da9c1015f4586f?, 0xb?, 0x0?, 0x0?, 0x13?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000f48600 sp=0xc000f485e0 pc=0x4380b6
runtime.netpollblock(0x47dba5?, 0x40482f?, 0x0?)
        /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc000f48638 sp=0xc000f48600 pc=0x4308b7
internal/poll.runtime_pollWait(0x7f8f1bfd5838, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc000f48658 sp=0xc000f48638 pc=0x462a09
internal/poll.(*pollDesc).wait(0xc0001d4980?, 0xc000568000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc000f48680 sp=0xc000f48658 pc=0x4a3bf2
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0001d4980, {0xc000568000, 0x1500, 0x1500})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc000f48718 sp=0xc000f48680 pc=0x4a4fd9
net.(*netFD).Read(0xc0001d4980, {0xc000568000?, 0xc000568cc9?, 0x3de?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc000f48760 sp=0xc000f48718 pc=0x5e6a69
net.(*conn).Read(0xc0108ea0e8, {0xc000568000?, 0x82718fa4ce0a740d?, 0xc0000b4fb0?})
        /usr/local/go/src/net/net.go:183 +0x45 fp=0xc000f487a8 sp=0xc000f48760 pc=0x5f8cc5
net.(*TCPConn).Read(0xc000f48840?, {0xc000568000?, 0xc01c8fa300?, 0x18?})
        <autogenerated>:1 +0x29 fp=0xc000f487d8 sp=0xc000f487a8 pc=0x60af69
crypto/tls.(*atLeastReader).Read(0xc01c8fa300, {0xc000568000?, 0xc01c8fa300?, 0x0?})
        /usr/local/go/src/crypto/tls/conn.go:788 +0x3d fp=0xc000f48820 sp=0xc000f487d8 pc=0x648e1d
bytes.(*Buffer).ReadFrom(0xc0000b5090, {0x1474240, 0xc01c8fa300})
        /usr/local/go/src/bytes/buffer.go:202 +0x98 fp=0xc000f48878 sp=0xc000f48820 pc=0x514cf8
crypto/tls.(*Conn).readFromUntil(0xc0000b4e00, {0x14784a0?, 0xc0108ea0e8}, 0x0?)
        /usr/local/go/src/crypto/tls/conn.go:810 +0xe5 fp=0xc000f488b8 sp=0xc000f48878 pc=0x649005
crypto/tls.(*Conn).readRecordOrCCS(0xc0000b4e00, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:617 +0x116 fp=0xc000f48c18 sp=0xc000f488b8 pc=0x6464f6
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc0000b4e00, {0xc000b8d000, 0x1000, 0x21?})
        /usr/local/go/src/crypto/tls/conn.go:1316 +0x16f fp=0xc000f48c88 sp=0xc000f48c18 pc=0x64c3ef
net/http.(*persistConn).Read(0xc0003da480, {0xc000b8d000?, 0xc000b79c20?, 0xc000f48d30?})
        /usr/local/go/src/net/http/transport.go:1943 +0x4e fp=0xc000f48ce8 sp=0xc000f48c88 pc=0x71562e
bufio.(*Reader).fill(0xc01fa64840)
        /usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc000f48d20 sp=0xc000f48ce8 pc=0x680dff
bufio.(*Reader).Peek(0xc01fa64840, 0x1)
        /usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc000f48d40 sp=0xc000f48d20 pc=0x680f5d
net/http.(*persistConn).readLoop(0xc0003da480)
        /usr/local/go/src/net/http/transport.go:2107 +0x1ac fp=0xc000f48fc8 sp=0xc000f48d40 pc=0x71644c
net/http.(*Transport).dialConn.func5()
        /usr/local/go/src/net/http/transport.go:1765 +0x26 fp=0xc000f48fe0 sp=0xc000f48fc8 pc=0x714be6
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000f48fe8 sp=0xc000f48fe0 pc=0x468561
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1765 +0x16ea

goroutine 8370 [IO wait, 562 minutes]:
runtime.gopark(0x5?, 0xb?, 0x1?, 0x0?, 0x13?)
        /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0146135d8 sp=0xc0146135b8 pc=0x4380b6
runtime.netpollblock(0x47dd65?, 0x40482f?, 0x0?)
        /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc014613610 sp=0xc0146135d8 pc=0x4308b7
internal/poll.runtime_pollWait(0x7f8f1bfd5838, 0x77)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc014613630 sp=0xc014613610 pc=0x462a09
internal/poll.(*pollDesc).wait(0xc0001d4980?, 0xc0145d20c4?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc014613658 sp=0xc014613630 pc=0x4a3bf2
internal/poll.(*pollDesc).waitWrite(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:93
internal/poll.(*FD).Write(0xc0001d4980, {0xc0145d0000, 0x4016, 0x4800})
        /usr/local/go/src/internal/poll/fd_unix.go:391 +0x2f6 fp=0xc014613708 sp=0xc014613658 pc=0x4a7216
net.(*netFD).Write(0xc0001d4980, {0xc0145d0000?, 0x80?, 0x114c3c0?})
        /usr/local/go/src/net/fd_posix.go:96 +0x29 fp=0xc014613750 sp=0xc014613708 pc=0x5e72e9
net.(*conn).Write(0xc0108ea0e8, {0xc0145d0000?, 0x8000?, 0xc0145d0000?})
        /usr/local/go/src/net/net.go:195 +0x45 fp=0xc014613798 sp=0xc014613750 pc=0x5f8ea5
net.(*TCPConn).Write(0x8000?, {0xc0145d0000?, 0x47326e?, 0xc010a00840?})
        <autogenerated>:1 +0x29 fp=0xc0146137c8 sp=0xc014613798 pc=0x60b269
crypto/tls.(*Conn).write(0xc0000b4e00, {0xc0145d0000?, 0x5?, 0x4800?})
        /usr/local/go/src/crypto/tls/conn.go:923 +0x10d fp=0xc014613818 sp=0xc0146137c8 pc=0x64972d
crypto/tls.(*Conn).writeRecordLocked(0xc0000b4e00, 0x17, {0xc013c70000, 0x8000, 0x8000})
        /usr/local/go/src/crypto/tls/conn.go:991 +0x354 fp=0xc0146138f0 sp=0xc014613818 pc=0x649bf4
crypto/tls.(*Conn).Write(0xc01c2e2b28?, {0xc013c70000, 0x8000, 0x8000})
        /usr/local/go/src/crypto/tls/conn.go:1186 +0x411 fp=0xc0146139e0 sp=0xc0146138f0 pc=0x64b1b1
io.copyBuffer({0x7f8f1c02f698, 0xc0000b4e00}, {0x14782a0, 0xc01c2e2b28}, {0x0, 0x0, 0x0})
        /usr/local/go/src/io/io.go:429 +0x204 fp=0xc014613a60 sp=0xc0146139e0 pc=0x49e8c4
io.Copy(...)
        /usr/local/go/src/io/io.go:386
net/http.persistConnWriter.ReadFrom({0xc01248b0b0?}, {0x14782a0?, 0xc01c2e2b28?})
        /usr/local/go/src/net/http/transport.go:1790 +0x5b fp=0xc014613ab8 sp=0xc014613a60 pc=0x7151bb
bufio.(*Writer).ReadFrom(0xc01d142780, {0x14782a0, 0xc01c2e2b28})
        /usr/local/go/src/bufio/bufio.go:783 +0x1ab fp=0xc014613b40 sp=0xc014613ab8 pc=0x6834eb
io.copyBuffer({0x1473400, 0xc01d142780}, {0x14782a0, 0xc01c2e2b28}, {0x0, 0x0, 0x0})
        /usr/local/go/src/io/io.go:413 +0x14b fp=0xc014613bc0 sp=0xc014613b40 pc=0x49e80b
io.Copy(...)
        /usr/local/go/src/io/io.go:386
net/http.(*transferWriter).doBodyCopy(0xc00f92c280, {0x1473400?, 0xc01d142780?}, {0x14782a0?, 0xc01c2e2b28?})
        /usr/local/go/src/net/http/transfer.go:412 +0x4d fp=0xc014613c20 sp=0xc014613bc0 pc=0x7089ed
net/http.(*transferWriter).writeBody(0xc00f92c280, {0x1473400, 0xc01d142780})
        /usr/local/go/src/net/http/transfer.go:370 +0x3e5 fp=0xc014613d18 sp=0xc014613c20 pc=0x708545
net/http.(*Request).write(0xc00f626f00, {0x1473400, 0xc01d142780}, 0x0, 0xc013be8de0, 0x0)
        /usr/local/go/src/net/http/request.go:712 +0xc06 fp=0xc014613ef0 sp=0xc014613d18 pc=0x6f0526
net/http.(*persistConn).writeLoop(0xc0003da480)
        /usr/local/go/src/net/http/transport.go:2413 +0x171 fp=0xc014613fc8 sp=0xc014613ef0 pc=0x718191
net/http.(*Transport).dialConn.func6()
        /usr/local/go/src/net/http/transport.go:1766 +0x26 fp=0xc014613fe0 sp=0xc014613fc8 pc=0x714b86
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc014613fe8 sp=0xc014613fe0 pc=0x468561
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1766 +0x173d

rax    0xca
rbx    0x0
rcx    0x46a483
rdx    0x0
rdi    0x1c94248
rsi    0x80
rbp    0x7ffcaebb4b38
rsp    0x7ffcaebb4af0
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0xc00004e000
r13    0x4
r14    0x1c93b20
r15    0x1
rip    0x46a481
rflags 0x286
cs     0x33
fs     0x0
gs     0x0

That goroutine is apparently stuck for 9+ hours waiting for an upload to complete. Probably, the network connection somehow dropped and nobody noticed… (Set timeouts for backend connections · Issue #4193 · restic/restic · GitHub might fix that problem, although I don’t see an indication why the connection would become stuck).

Which restic version are you using?

It was using 0.16.0.