I’ve just found that restic now uses pretty large amount of RAM. (Around 2.5GB). This should be ok for modern computers, but seems to much for something small and low end (like Raspberry Pi).
Does RAM usage depends mostly on repository size or directory that is being backed up?
My current repository size is around 1TB, shared via rest-server. And restic is running on virtual machine with ~30GB of disk space
In my experience, RAM usage depends mostly on the number of files that are backed up. Each file in the repository is described by a JSON structure, and restic uses RAM to first parse then keep that data around. And in some cases (prune) it also might depend on the number of snapshots.
@fd0 might give better answer, 'cause he did some memory usage optimizations recently, so he has better picture what’s going on there.
I’ve actually tried to backup Raspberry Pi using existing 1TB repository (shared via rest-server). And got followed.
At the same time backup to new repository works even after a few backups.
runtime: out of memory: cannot allocate 536870912-byte block (572522496 in use)
fatal error: out of memory
runtime stack:
runtime.throw(0x4a6c44, 0xd)
/usr/local/go/src/runtime/panic.go:596 +0x70
runtime.largeAlloc(0x1ffffe00, 0x76f4c701, 0x10a524b0)
/usr/local/go/src/runtime/malloc.go:809 +0xe0
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:702 +0x30
runtime.systemstack(0x10a524d0)
/usr/local/go/src/runtime/asm_arm.s:264 +0x80
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1132
goroutine 87 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_arm.s:209 +0x4 fp=0x10a61a1c sp=0x10a61a18
runtime.mallocgc(0x1ffffe00, 0x42de98, 0x10b3c101, 0x0)
/usr/local/go/src/runtime/malloc.go:703 +0x9e0 fp=0x10a61a78 sp=0x10a61a1c
runtime.makeslice(0x42de98, 0x1ffffe00, 0x1ffffe00, 0x10c003f0, 0xb750c, 0x10aaefe4)
/usr/local/go/src/runtime/slice.go:54 +0x68 fp=0x10a61a9c sp=0x10a61a78
bytes.makeSlice(0x1ffffe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:201 +0x58 fp=0x10a61ab8 sp=0x10a61a9c
bytes.(*Buffer).ReadFrom(0x10a61b14, 0x76ef1058, 0x10aaefd0, 0x200, 0x0, 0x10c7a000, 0x0)
/usr/local/go/src/bytes/buffer.go:173 +0x270 fp=0x10a61af0 sp=0x10a61ab8
io/ioutil.readAll(0x76ef1058, 0x10aaefd0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:33 +0xd0 fp=0x10a61b68 sp=0x10a61af0
io/ioutil.ReadAll(0x76ef1058, 0x10aaefd0, 0x10aaefd0, 0x76ef1058, 0x10aaefd0, 0x10b487c0, 0x40)
/usr/local/go/src/io/ioutil/ioutil.go:42 +0x34 fp=0x10a61b90 sp=0x10a61b68
restic/backend.LoadAll(0x6b44b0, 0x10a0a19c, 0x6b5c90, 0x10b03720, 0x4a26f2, 0x5, 0x10b487c0, 0x40, 0x0, 0x0, ...)
src/restic/backend/utils.go:29 +0x114 fp=0x10a61bd4 sp=0x10a61b90
restic/repository.(*Repository).LoadAndDecrypt(0x10d4bbf0, 0x6b44b0, 0x10a0a19c, 0x4a26f2, 0x5, 0xf232e7fb, 0xf42bf696, 0xcfa71c4d, 0x4c0152f3, 0xe5abf08f, ...)
src/restic/repository/repository.go:58 +0x108 fp=0x10a61cfc sp=0x10a61bd4
restic/repository.LoadIndexWithDecoder(0x6b44b0, 0x10a0a19c, 0x6b69d0, 0x10d4bbf0, 0xf232e7fb, 0xf42bf696, 0xcfa71c4d, 0x4c0152f3, 0xe5abf08f, 0xf3ff4268, ...)
src/restic/repository/index.go:526 +0xc4 fp=0x10a61db0 sp=0x10a61cfc
restic/repository.LoadIndex(0x6b44b0, 0x10a0a19c, 0x6b69d0, 0x10d4bbf0, 0xf232e7fb, 0xf42bf696, 0xcfa71c4d, 0x4c0152f3, 0xe5abf08f, 0xf3ff4268, ...)
src/restic/repository/repository.go:353 +0x58 fp=0x10a61e0c sp=0x10a61db0
restic/repository.(*Repository).LoadIndex.func1(0x6b44b0, 0x10a0a19c, 0xf232e7fb, 0xf42bf696, 0xcfa71c4d, 0x4c0152f3, 0xe5abf08f, 0xf3ff4268, 0x9e98dd76, 0x4f9ba53f, ...)
src/restic/repository/repository.go:321 +0x5c fp=0x10a61ea4 sp=0x10a61e0c
restic/repository.ParallelWorkFuncParseID.func1(0x6b44b0, 0x10a0a19c, 0x10afe7c0, 0x40, 0x1, 0x0)
src/restic/repository/parallel.go:74 +0x14c fp=0x10a61f54 sp=0x10a61ea4
restic/repository.FilesInParallel.func1(0x10bb41f0, 0x10b48380, 0x10bf20b8, 0x6b44b0, 0x10a0a19c, 0x10afe800)
src/restic/repository/parallel.go:39 +0xf8 fp=0x10a61fd4 sp=0x10a61f54
runtime.goexit()
/usr/local/go/src/runtime/asm_arm.s:1017 +0x4 fp=0x10a61fd4 sp=0x10a61fd4
created by restic/repository.FilesInParallel
src/restic/repository/parallel.go:48 +0xf0
goroutine 1 [chan receive]:
restic/repository.(*Repository).LoadIndex(0x10d4bbf0, 0x6b44b0, 0x10a0a19c, 0x0, 0x0)
src/restic/repository/repository.go:340 +0x134
main.runBackup(0x0, 0x0, 0x0, 0x10aac500, 0x1d, 0x20, 0x0, 0x0, 0x0, 0x0, ...)
src/cmds/restic/cmd_backup.go:376 +0x33c
main.glob..func2(0x6db5e8, 0x10b341e0, 0x1, 0x3b, 0x0, 0x0)
src/cmds/restic/cmd_backup.go:39 +0xcc
github.com/spf13/cobra.(*Command).execute(0x6db5e8, 0x10b34000, 0x3b, 0x3c, 0x6db5e8, 0x10b34000)
src/github.com/spf13/cobra/command.go:647 +0x308
github.com/spf13/cobra.(*Command).ExecuteC(0x6db3b8, 0x5401, 0x10b13ee0, 0x0)
src/github.com/spf13/cobra/command.go:726 +0x220
github.com/spf13/cobra.(*Command).Execute(0x6db3b8, 0x24, 0x10b13f5c)
src/github.com/spf13/cobra/command.go:685 +0x1c
main.main()
src/cmds/restic/main.go:63 +0x148
goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:116 +0x154
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x14
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x30
goroutine 6 [chan receive]:
restic.init.1.func1.1()
src/restic/lock.go:263 +0x9c
created by restic.init.1.func1
src/restic/lock.go:266 +0x24
goroutine 7 [select, locked to thread]:
runtime.gopark(0x4c5254, 0x0, 0x4a2f04, 0x6, 0x18, 0x2)
/usr/local/go/src/runtime/proc.go:271 +0xfc
runtime.selectgoImpl(0x10a22fa0, 0x0, 0xc)
/usr/local/go/src/runtime/select.go:423 +0x116c
runtime.selectgo(0x10a22fa0)
/usr/local/go/src/runtime/select.go:238 +0x10
runtime.ensureSigM.func1()
/usr/local/go/src/runtime/signal_unix.go:434 +0x2bc
runtime.goexit()
/usr/local/go/src/runtime/asm_arm.s:1017 +0x4
goroutine 8 [chan receive]:
restic.init.2.func1(0x10a46280)
src/restic/progress_unix.go:17 +0x38
created by restic.init.2
src/restic/progress_unix.go:21 +0x90
goroutine 9 [chan receive]:
main.CleanupHandler(0x10aff840)
src/cmds/restic/cleanup.go:62 +0x38
created by main.init.1
src/cmds/restic/cleanup.go:25 +0x90
goroutine 14 [select]:
restic/backend/rest.(*restBackend).List.func2(0x10d62fc0, 0x10afd570, 0x6b44b0, 0x10a0a19c)
src/restic/backend/rest/rest.go:335 +0x134
created by restic/backend/rest.(*restBackend).List
src/restic/backend/rest/rest.go:341 +0x29c
goroutine 19 [runnable]:
crypto/cipher.(*gcm).mul(0x10cfb7a0, 0x10a3baa8)
/usr/local/go/src/crypto/cipher/gcm.go:246 +0x148
crypto/cipher.(*gcm).updateBlocks(0x10cfb7a0, 0x10a3baa8, 0x12da200d, 0x4000, 0x7ff3)
/usr/local/go/src/crypto/cipher/gcm.go:284 +0x1a0
crypto/cipher.(*gcm).update(0x10cfb7a0, 0x10a3baa8, 0x12da200d, 0x4000, 0x7ff3)
/usr/local/go/src/crypto/cipher/gcm.go:293 +0x50
crypto/cipher.(*gcm).auth(0x10cfb7a0, 0x10a3bb00, 0x10, 0x10, 0x12da200d, 0x4000, 0x7ff3, 0x10b460d0, 0xd, 0xd, ...)
/usr/local/go/src/crypto/cipher/gcm.go:376 +0x7c
crypto/cipher.(*gcm).Open(0x10cfb7a0, 0x12da200d, 0x0, 0x7ff3, 0x10d55260, 0xc, 0xc, 0x12da200d, 0x4000, 0x7ff3, ...)
/usr/local/go/src/crypto/cipher/gcm.go:184 +0x1f0
crypto/tls.(*fixedNonceAEAD).Open(0x10d55260, 0x12da200d, 0x0, 0x7ff3, 0x12da2005, 0x8, 0x7ffb, 0x12da200d, 0x4010, 0x7ff3, ...)
/usr/local/go/src/crypto/tls/cipher_suites.go:180 +0xac
crypto/tls.(*halfConn).decrypt(0x10b460a0, 0x10b3c1a0, 0x401d, 0x10b3c1a0, 0x10b3c180)
/usr/local/go/src/crypto/tls/conn.go:306 +0x53c
crypto/tls.(*Conn).readRecord(0x10b46000, 0x4c5217, 0x10b460a0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:647 +0x204
crypto/tls.(*Conn).Read(0x10b46000, 0x10d6d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1134 +0xdc
bufio.(*Reader).Read(0x10d4baa0, 0x10b43720, 0x9, 0x9, 0x1181c, 0x1cbb0, 0x10aaeffc)
/usr/local/go/src/bufio/bufio.go:213 +0x280
io.ReadAtLeast(0x6b0ad0, 0x10d4baa0, 0x10b43720, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0)
/usr/local/go/src/io/io.go:307 +0x8c
io.ReadFull(0x6b0ad0, 0x10d4baa0, 0x10b43720, 0x9, 0x9, 0x1dc338, 0x10aaefe4, 0x10ce61e0)
/usr/local/go/src/io/io.go:325 +0x40
net/http.http2readFrameHeader(0x10b43720, 0x9, 0x9, 0x6b0ad0, 0x10d4baa0, 0x0, 0x0, 0x0, 0x10b38000, 0x4000)
/usr/local/go/src/net/http/h2_bundle.go:781 +0x50
net/http.(*http2Framer).ReadFrame(0x10b43700, 0x10ca45c0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/h2_bundle.go:1008 +0x74
net/http.(*http2clientConnReadLoop).run(0x10a3bfd8, 0x4c4e64, 0x10a1cfd8)
/usr/local/go/src/net/http/h2_bundle.go:6637 +0x60
net/http.(*http2ClientConn).readLoop(0x10c00300)
/usr/local/go/src/net/http/h2_bundle.go:6566 +0x8c
created by net/http.(*http2Transport).newClientConn
/usr/local/go/src/net/http/h2_bundle.go:5892 +0x59c
goroutine 23 [semacquire]:
sync.runtime_Semacquire(0x10bb41fc)
/usr/local/go/src/runtime/sema.go:47 +0x24
sync.(*WaitGroup).Wait(0x10bb41f0)
/usr/local/go/src/sync/waitgroup.go:131 +0x94
restic/repository.FilesInParallel(0x6b44b0, 0x10a0a19c, 0x76eb53a0, 0x10b03720, 0x4a26f2, 0x5, 0x14, 0x10bf20b8, 0x0, 0x0)
src/restic/repository/parallel.go:51 +0x10c
restic/repository.(*Repository).LoadIndex.func2(0x10b48340, 0x10b48300, 0x6b44b0, 0x10a0a19c, 0x10d4bbf0, 0x10bb41e0)
src/restic/repository/repository.go:337 +0xac
created by restic/repository.(*Repository).LoadIndex
src/restic/repository/repository.go:338 +0x110
goroutine 22 [select]:
main.refreshLocks(0x6de6b8, 0x10b482c0)
src/cmds/restic/lock.go:71 +0x22c
created by main.lockRepository
src/cmds/restic/lock.go:48 +0x294Restic uses about 2.5 GB of RAM for me, while backing up to a fairly old repository (about 150-200 GB in size).
I’ve solved this by using ‘rsync’ to copy all data from Raspberry Pi to better machine and then using restic to backup result dir.
FYI, there was a recent commit that may solve the excessive memory usage no the Pi, see https://github.com/restic/restic/issues/1256 and https://github.com/restic/restic/pull/1267
Hi,
Not tried it yet. But probably will not help me, since I don’t use S3 at all.
Btw, I’m waiting for restic prune to finish (after pull’ing metadata cache changes). And it looks like sooner or later I’ll be unable to run restic prune at all…
It already takes ~9GB of RAM (on amd64 box):
%ps -eo vsz,rsz,pid,cmd --sort rsz | grep restic
9226216 9113448 20764 restic prune
Current repo size is around 1.3TB:
% sudo du -h --max-depth=1 /srv/backup/restserver
8.0K /srv/backup/restserver/locks
914M /srv/backup/restserver/index
20K /srv/backup/restserver/keys
1.3T /srv/backup/restserver/data
696K /srv/backup/restserver/snapshots
1.3T /srv/backup/restserver
Regular restic backup call requires ~2.5-3GB of RAM. I can show usage later (after restic prune)
Ok. restic prune is done. So i’ve just started backup on another machine (repo is same 1.3TB shared via restserver). I’ve waited for progress indicator and captured just after this:
%ps -eo vsz,rsz,pid,cmd --sort rsz | grep restic
4029960 3809500 2556 /usr/local/bin/restic backup --exclude-caches...
So it’s around 3.8GB of RAM