Runtime: out of memory (fatal error: out of memory)

Support
#1

I am running the cli version 2.7.2 for arm on my QNAP TS-431+
duplicacy is running without any problems since years. But now I got this problem:

Uploaded chunk 10707 size 4353716, 8.24MB/s 02:16:36 43.9%
Uploaded chunk 10712 size 4857424, 8.23MB/s 02:16:44 43.9%
runtime: out of memory: cannot allocate 8568832-byte block (1797750784 in use)
fatal error: out of memory

runtime stack:
runtime.throw(0x9f8c1d, 0xd)
        /usr/local/go/src/runtime/panic.go:617 +0x5c
runtime.largeAlloc(0x82a2c5, 0x101, 0x247b960)
        /usr/local/go/src/runtime/malloc.go:1057 +0x140
runtime.mallocgc.func1()
        /usr/local/go/src/runtime/malloc.go:950 +0x38
runtime.systemstack(0x24dcd20)
        /usr/local/go/src/runtime/asm_arm.s:354 +0x84
runtime.mstart()
        /usr/local/go/src/runtime/proc.go:1153

goroutine 38 [running]:
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_arm.s:298 +0x4 fp=0x68ddc20 sp=0x68ddc1c pc=0x6fe98
runtime.mallocgc(0x82a2c5, 0x8e44b8, 0x1, 0x128b8990)
        /usr/local/go/src/runtime/malloc.go:949 +0x8c4 fp=0x68ddc88 sp=0x68ddc20 pc=0x1b8e8
runtime.makeslice(0x8e44b8, 0x82a2c5, 0x82a2c5, 0x1)
        /usr/local/go/src/runtime/slice.go:49 +0x70 fp=0x68ddc9c sp=0x68ddc88 pc=0x57a7c
github.com/gilbertchen/duplicacy/vendor/github.com/bkaradzic/go-lz4.Encode(0x4440001a, 0x0, 0x2b3deb8, 0x22800000, 0x82200f, 0x1000000, 0x0, 0x0, 0x0, 0x65a8a778, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/vendor/github.com/bkaradzic/go-lz4/writer.go:112 +0x490 fp=0x68ddd00 sp=0x68ddc9c pc=0x3b5b40
github.com/gilbertchen/duplicacy/src.(*Chunk).Encrypt(0x26ae2d0, 0x290b400, 0x20, 0x40, 0x68dded8, 0x20, 0x0, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunk.go:272 +0x584 fp=0x68dde90 sp=0x68ddd00 pc=0x7c28f0
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Upload(0x26ae210, 0x1, 0x26ae2d0, 0x29d9, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:131 +0x154 fp=0x68ddf8c sp=0x68dde90 pc=0x7cb8d8
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start.func1(0x26ae210, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:60 +0x54 fp=0x68ddfe4 sp=0x68ddf8c pc=0x823ab8
runtime.goexit()
        /usr/local/go/src/runtime/asm_arm.s:868 +0x4 fp=0x68ddfe4 sp=0x68ddfe4 pc=0x71bd0
created by github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:55 +0x38

goroutine 1 [runnable]:
os.(*File).Read(0x1209cdb0, 0x204fd95f, 0x100000, 0x1026a1, 0xfd95f, 0x0, 0x0)
        /usr/local/go/src/os/file.go:104 +0x234
github.com/gilbertchen/duplicacy/src.(*ChunkMaker).ForEachChunk(0x2948000, 0xbe1e48, 0x1209cdb0, 0x29f0768, 0x29f0678)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkmaker.go:200 +0x1c0
github.com/gilbertchen/duplicacy/src.(*BackupManager).Backup(0x25e40f0, 0x2428084, 0x18, 0x1, 0x5, 0x0, 0x0, 0x1, 0x0, 0x0, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_backupmanager.go:531 +0x3148
main.backupRepository(0x26c8120)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/duplicacy/duplicacy_main.go:784 +0x4a0
github.com/gilbertchen/duplicacy/vendor/github.com/gilbertchen/cli.Command.Run(0x9f1033, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa22878, 0x30, 0x0, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/vendor/github.com/gilbertchen/cli/command.go:160 +0x5c4
github.com/gilbertchen/duplicacy/vendor/github.com/gilbertchen/cli.(*App).Run(0x24bee60, 0x24840c0, 0x7, 0x8, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/vendor/github.com/gilbertchen/cli/app.go:179 +0x4cc
main.main()
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/duplicacy/duplicacy_main.go:2201 +0x4878

goroutine 18 [syscall, 109 minutes]:
os/signal.signal_recv(0x0)
        /usr/local/go/src/runtime/sigqueue.go:139 +0x130
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:23 +0x14
created by os/signal.init.0
        /usr/local/go/src/os/signal/signal_unix.go:29 +0x30

goroutine 4 [chan receive, 109 minutes]:
main.main.func1(0x269b8c0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/duplicacy/duplicacy_main.go:2195 +0x64
created by main.main
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/duplicacy/duplicacy_main.go:2194 +0x4848

goroutine 21 [select]:
github.com/gilbertchen/duplicacy/vendor/go.opencensus.io/stats/view.(*worker).start(0x25cae00)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/vendor/go.opencensus.io/stats/view/worker.go:154 +0xb0
created by github.com/gilbertchen/duplicacy/vendor/go.opencensus.io/stats/view.init.0
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/vendor/go.opencensus.io/stats/view/worker.go:32 +0x48

goroutine 41 [select]:
net/http.(*persistConn).roundTrip(0x11d4e0a0, 0x1187b560, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:2194 +0x5b8
net/http.(*Transport).roundTrip(0x12f5638, 0xf8c6b00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:481 +0x798
net/http.(*Transport).RoundTrip(0x12f5638, 0xf8c6b00, 0x12f5638, 0x0, 0x0)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x24
net/http.send(0xf8c6b00, 0xbe1c08, 0x12f5638, 0x0, 0x0, 0x0, 0x0, 0x0, 0x11bca5d0, 0x658f3c2a, ...)
        /usr/local/go/src/net/http/client.go:250 +0x3b8
net/http.(*Client).send(0x12fb2a8, 0xf8c6b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x24, 0x80, ...)
        /usr/local/go/src/net/http/client.go:174 +0xbc
net/http.(*Client).do(0x12fb2a8, 0xf8c6b00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/client.go:641 +0x1d4
net/http.(*Client).Do(...)
        /usr/local/go/src/net/http/client.go:509
github.com/gilbertchen/duplicacy/src.(*B2Client).call(0x26c8750, 0x4, 0xd9be460, 0x6a, 0x9ef4de, 0x4, 0x2731db4, 0x966950, 0x11bb8a80, 0x16, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:218 +0x15c
github.com/gilbertchen/duplicacy/src.(*B2Client).UploadFile(0x26c8750, 0x4, 0xdc6de00, 0x48, 0x49000000, 0x127b33e, 0x22f426e, 0x0, 0x2731ed8, 0x110895e0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:635 +0x404
github.com/gilbertchen/duplicacy/src.(*B2Storage).UploadFile(0x27b3380, 0x4, 0xdc6de00, 0x48, 0x49000000, 0x127b33e, 0x22f426e, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2storage.go:220 +0x94
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Upload(0x26ae210, 0x4, 0x27f9650, 0x29d5, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:138 +0x1ec
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start.func1(0x26ae210, 0x4)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:60 +0x54
created by github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:55 +0x38

goroutine 40 [select]:
net/http.(*persistConn).roundTrip(0x928df40, 0x1187a500, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:2194 +0x5b8
net/http.(*Transport).roundTrip(0x12f5638, 0xf8c6a00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:481 +0x798
net/http.(*Transport).RoundTrip(0x12f5638, 0xf8c6a00, 0x12f5638, 0x0, 0x0)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x24
net/http.send(0xf8c6a00, 0xbe1c08, 0x12f5638, 0x0, 0x0, 0x0, 0x0, 0x0, 0x110a7898, 0x658f3c2a, ...)
        /usr/local/go/src/net/http/client.go:250 +0x3b8
net/http.(*Client).send(0x12fb2a8, 0xf8c6a00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x24, 0x80, ...)
        /usr/local/go/src/net/http/client.go:174 +0xbc
net/http.(*Client).do(0x12fb2a8, 0xf8c6a00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/client.go:641 +0x1d4
net/http.(*Client).Do(...)
        /usr/local/go/src/net/http/client.go:509
github.com/gilbertchen/duplicacy/src.(*B2Client).call(0x26c8750, 0x3, 0x881d500, 0x6a, 0x9ef4de, 0x4, 0x244ddb4, 0x966950, 0x114e5a10, 0x16, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:218 +0x15c
github.com/gilbertchen/duplicacy/src.(*B2Client).UploadFile(0x26c8750, 0x3, 0xe156cd0, 0x48, 0x4d000000, 0xd36208, 0x228f3a2, 0x0, 0x244ded8, 0x107d3680)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:635 +0x404
github.com/gilbertchen/duplicacy/src.(*B2Storage).UploadFile(0x27b3380, 0x3, 0xe156cd0, 0x48, 0x4d000000, 0xd36208, 0x228f3a2, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2storage.go:220 +0x94
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Upload(0x26ae210, 0x3, 0x26ae270, 0x29d4, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:138 +0x1ec
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start.func1(0x26ae210, 0x3)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:60 +0x54
created by github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:55 +0x38

goroutine 39 [select]:
net/http.(*persistConn).roundTrip(0x10a121e0, 0x10b2aa20, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:2194 +0x5b8
net/http.(*Transport).roundTrip(0x12f5638, 0xf3dc500, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:481 +0x798
net/http.(*Transport).RoundTrip(0x12f5638, 0xf3dc500, 0x12f5638, 0x0, 0x0)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x24
net/http.send(0xf3dc500, 0xbe1c08, 0x12f5638, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1086b000, 0x658f2299, ...)
        /usr/local/go/src/net/http/client.go:250 +0x3b8
net/http.(*Client).send(0x12fb2a8, 0xf3dc500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x24, 0x80, ...)
        /usr/local/go/src/net/http/client.go:174 +0xbc
net/http.(*Client).do(0x12fb2a8, 0xf3dc500, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/client.go:641 +0x1d4
net/http.(*Client).Do(...)
        /usr/local/go/src/net/http/client.go:509
github.com/gilbertchen/duplicacy/src.(*B2Client).call(0x26c8750, 0x2, 0x98d9f10, 0x6a, 0x9ef4de, 0x4, 0x68dbdb4, 0x966950, 0x10a1dbc0, 0x16, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:218 +0x15c
github.com/gilbertchen/duplicacy/src.(*B2Client).UploadFile(0x26c8750, 0x2, 0xe03d4f0, 0x48, 0x3e800000, 0x11187a3, 0x22b7908, 0x0, 0x68dbed8, 0x101b2460)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:635 +0x404
github.com/gilbertchen/duplicacy/src.(*B2Storage).UploadFile(0x27b3380, 0x2, 0xe03d4f0, 0x48, 0x3e800000, 0x11187a3, 0x22b7908, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2storage.go:220 +0x94
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Upload(0x26ae210, 0x2, 0x26ae2a0, 0x29cf, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:138 +0x1ec
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start.func1(0x26ae210, 0x2)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:60 +0x54
created by github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:55 +0x38

goroutine 37 [select]:
net/http.(*persistConn).roundTrip(0x11d4e140, 0x125c24e0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:2194 +0x5b8
net/http.(*Transport).roundTrip(0x12f5638, 0xf8c6c00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/transport.go:481 +0x798
net/http.(*Transport).RoundTrip(0x12f5638, 0xf8c6c00, 0x12f5638, 0x0, 0x0)
        /usr/local/go/src/net/http/roundtrip.go:17 +0x24
net/http.send(0xf8c6c00, 0xbe1c08, 0x12f5638, 0x0, 0x0, 0x0, 0x0, 0x0, 0x11bcb3c0, 0x658f3c2b, ...)
        /usr/local/go/src/net/http/client.go:250 +0x3b8
net/http.(*Client).send(0x12fb2a8, 0xf8c6c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x24, 0x80, ...)
        /usr/local/go/src/net/http/client.go:174 +0xbc
net/http.(*Client).do(0x12fb2a8, 0xf8c6c00, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/client.go:641 +0x1d4
net/http.(*Client).Do(...)
        /usr/local/go/src/net/http/client.go:509
github.com/gilbertchen/duplicacy/src.(*B2Client).call(0x26c8750, 0x0, 0xe377b20, 0x6a, 0x9ef4de, 0x4, 0x68e1db4, 0x966950, 0x1256c630, 0x16, ...)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:218 +0x15c
github.com/gilbertchen/duplicacy/src.(*B2Client).UploadFile(0x26c8750, 0x0, 0xde4ff40, 0x48, 0x1e2bc000, 0x1c0c74, 0x13e37b0, 0x0, 0x68e1ed8, 0x11089e20)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2client.go:635 +0x404
github.com/gilbertchen/duplicacy/src.(*B2Storage).UploadFile(0x27b3380, 0x0, 0xde4ff40, 0x48, 0x1e2bc000, 0x1c0c74, 0x13e37b0, 0x0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_b2storage.go:220 +0x94
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Upload(0x26ae210, 0x0, 0x51badb0, 0x29d7, 0x1)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:138 +0x1ec
github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start.func1(0x26ae210, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:60 +0x54
created by github.com/gilbertchen/duplicacy/src.(*ChunkUploader).Start
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkuploader.go:55 +0x38

goroutine 50 [select, 107 minutes]:
github.com/gilbertchen/duplicacy/src.CreateChunkDownloader.func1(0x28360e0, 0x0)
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkdownloader.go:85 +0xcc
created by github.com/gilbertchen/duplicacy/src.CreateChunkDownloader
        /Users/chgang/zincbox/go/src/github.com/gilbertchen/duplicacy/src/duplicacy_chunkdownloader.go:82 +0x1ec

goroutine 18911 [IO wait]:
internal/poll.runtime_pollWait(0x65bd2d08, 0x72, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0x107bbf04, 0x72, 0x1500, 0x1567, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0x107bbef0, 0x10780000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x184
net.(*netFD).Read(0x107bbef0, 0x10780000, 0x1567, 0x1567, 0x18b, 0x1bba8, 0x120ad8b0)
        /usr/local/go/src/net/fd_unix.go:202 +0x38
net.(*conn).Read(0x11bcb480, 0x10780000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x5c
crypto/tls.(*atLeastReader).Read(0x120ad8b0, 0x10780000, 0x1567, 0x1567, 0x18384, 0x1f2188, 0xbe1b90)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x78
bytes.(*Buffer).ReadFrom(0x12666160, 0xbe0c90, 0x120ad8b0, 0x658f6458, 0x1f0b54, 0x912678, 0x9d8128)
        /usr/local/go/src/bytes/buffer.go:207 +0xac
crypto/tls.(*Conn).readFromUntil(0x12666000, 0xbe1b90, 0x11bcb480, 0x5, 0x11bcb480, 0x248202c)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0x12666000, 0xa38800, 0x126660b0, 0x2426525)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x108
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0x12666000, 0x126b5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x140
net/http.(*persistConn).Read(0x11d4e140, 0x126b5000, 0x1000, 0x1000, 0x1, 0x2bb44c, 0x4)
        /usr/local/go/src/net/http/transport.go:1526 +0x1c0
bufio.(*Reader).fill(0x1256d740)
        /usr/local/go/src/bufio/bufio.go:100 +0x114
bufio.(*Reader).Peek(0x1256d740, 0x1, 0x1c94e400, 0x7cf469c, 0x101ceec0, 0x1c9a8200, 0x101cee80)
        /usr/local/go/src/bufio/bufio.go:138 +0x38
net/http.(*persistConn).readLoop(0x11d4e140)
        /usr/local/go/src/net/http/transport.go:1679 +0x168
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1359 +0x924

goroutine 18787 [IO wait]:
internal/poll.runtime_pollWait(0x65bd2e94, 0x72, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0x107bb964, 0x72, 0x1500, 0x1567, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0x107bb950, 0x106c1000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x184
net.(*netFD).Read(0x107bb950, 0x106c1000, 0x1567, 0x1567, 0x173, 0x1bba8, 0x11bc7730)
        /usr/local/go/src/net/fd_unix.go:202 +0x38
net.(*conn).Read(0x1086bd60, 0x106c1000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x5c
crypto/tls.(*atLeastReader).Read(0x11bc7730, 0x106c1000, 0x1567, 0x1567, 0x18384, 0x1f2188, 0xbe1b90)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x78
bytes.(*Buffer).ReadFrom(0x102acb60, 0xbe0c90, 0x11bc7730, 0x65ba69ee, 0x1f0b54, 0x912678, 0x9d8128)
        /usr/local/go/src/bytes/buffer.go:207 +0xac
crypto/tls.(*Conn).readFromUntil(0x102aca00, 0xbe1b90, 0x1086bd60, 0x5, 0x1086bd60, 0x248202c)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0x102aca00, 0xa38800, 0x102acab0, 0x2426525)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x108
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0x102aca00, 0x12306000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x140
net/http.(*persistConn).Read(0x11d4e0a0, 0x12306000, 0x1000, 0x1000, 0x1, 0x2bb44c, 0x4)
        /usr/local/go/src/net/http/transport.go:1526 +0x1c0
bufio.(*Reader).fill(0x11bb9470)
        /usr/local/go/src/bufio/bufio.go:100 +0x114
bufio.(*Reader).Peek(0x11bb9470, 0x1, 0x1c962c80, 0x7e0369c, 0xecc1880, 0x1ca2a7c0, 0xecc1840)
        /usr/local/go/src/bufio/bufio.go:138 +0x38
net/http.(*persistConn).readLoop(0x11d4e0a0)
        /usr/local/go/src/net/http/transport.go:1679 +0x168
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1359 +0x924

goroutine 18912 [select]:
net/http.(*persistConn).writeLoop(0x11d4e140)
        /usr/local/go/src/net/http/transport.go:1978 +0xb8
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1360 +0x940

goroutine 18799 [select]:
net/http.(*persistConn).writeLoop(0x10a12460)
        /usr/local/go/src/net/http/transport.go:1978 +0xb8
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1360 +0x940

goroutine 18798 [IO wait]:
internal/poll.runtime_pollWait(0x65bd3020, 0x72, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0x107bbb44, 0x72, 0x1600, 0x16ab, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0x107bbb30, 0x106c2800, 0x16ab, 0x16ab, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x184
net.(*netFD).Read(0x107bbb30, 0x106c2800, 0x16ab, 0x16ab, 0x128, 0x1bba8, 0x127b9280)
        /usr/local/go/src/net/fd_unix.go:202 +0x38
net.(*conn).Read(0x11bcaac0, 0x106c2800, 0x16ab, 0x16ab, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x5c
crypto/tls.(*atLeastReader).Read(0x127b9280, 0x106c2800, 0x16ab, 0x16ab, 0x18384, 0x1f2188, 0xbe1b90)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x78
bytes.(*Buffer).ReadFrom(0xf58df60, 0xbe0c90, 0x127b9280, 0x49, 0x1f0b54, 0x912678, 0x9d8128)
        /usr/local/go/src/bytes/buffer.go:207 +0xac
crypto/tls.(*Conn).readFromUntil(0xf58de00, 0xbe1b90, 0x11bcaac0, 0x5, 0x11bcaac0, 0x658fba20)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0xf58de00, 0xa38800, 0xf58deb0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x108
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xf58de00, 0x1267d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x140
net/http.(*persistConn).Read(0x10a12460, 0x1267d000, 0x1000, 0x1000, 0x7cdef24, 0x2, 0x2)
        /usr/local/go/src/net/http/transport.go:1526 +0x1c0
bufio.(*Reader).fill(0x1256d290)
        /usr/local/go/src/bufio/bufio.go:100 +0x114
bufio.(*Reader).Peek(0x1256d290, 0x1, 0x2, 0x0, 0x0, 0x1c94e700, 0x0)
        /usr/local/go/src/bufio/bufio.go:138 +0x38
net/http.(*persistConn).readLoop(0x10a12460)
        /usr/local/go/src/net/http/transport.go:1679 +0x168
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1359 +0x924

goroutine 18788 [IO wait]:
internal/poll.runtime_pollWait(0x65bd2e94, 0x77, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0x107bb964, 0x77, 0x300, 0x30a, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitWrite(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:96
internal/poll.(*FD).Write(0x107bb950, 0x106d6c00, 0x101d, 0x101d, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:276 +0x230
net.(*netFD).Write(0x107bb950, 0x106d6c00, 0x101d, 0x101d, 0x1018, 0x0, 0x0)
        /usr/local/go/src/net/fd_unix.go:220 +0x38
net.(*conn).Write(0x1086bd60, 0x106d6c00, 0x101d, 0x101d, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:189 +0x5c
crypto/tls.(*Conn).write(0x102aca00, 0x106d6c00, 0x101d, 0x101d, 0x12307000, 0x1000, 0x1000)
        /usr/local/go/src/crypto/tls/conn.go:896 +0x118
crypto/tls.(*Conn).writeRecordLocked(0x102aca00, 0x17, 0x12307000, 0x1000, 0x1000, 0x11bb8a80, 0x12307000, 0x1000)
        /usr/local/go/src/crypto/tls/conn.go:945 +0x1fc
crypto/tls.(*Conn).Write(0x102aca00, 0x12307000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1114 +0x17c
net/http.persistConnWriter.Write(0x11d4e0a0, 0x12307000, 0x1000, 0x1000, 0x12307000, 0x1000, 0x1000)
        /usr/local/go/src/net/http/transport.go:1375 +0x40
bufio.(*Writer).Flush(0x1187baa0, 0x12307000, 0x1000)
        /usr/local/go/src/bufio/bufio.go:590 +0x64
bufio.(*Writer).ReadFrom(0x1187baa0, 0xbe1a10, 0x11bc7750, 0x65cd3cf8, 0x1187baa0, 0x2432001, 0x243203c)
        /usr/local/go/src/bufio/bufio.go:716 +0x270
io.copyBuffer(0xbe0b70, 0x1187baa0, 0xbe1a10, 0x11bc7750, 0x0, 0x0, 0x0, 0x1000, 0x1ea, 0x0, ...)
        /usr/local/go/src/io/io.go:388 +0x2f0
io.Copy(...)
        /usr/local/go/src/io/io.go:364
net/http.(*transferWriter).writeBody(0x107bb9f0, 0xbe0b70, 0x1187baa0, 0x2, 0x2)
        /usr/local/go/src/net/http/transfer.go:369 +0x610
net/http.(*Request).write(0xf8c6b00, 0xbe0b70, 0x1187baa0, 0x0, 0x1187bac0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/request.go:655 +0x578
net/http.(*persistConn).writeLoop(0x11d4e0a0)
        /usr/local/go/src/net/http/transport.go:1981 +0x134
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1360 +0x940

goroutine 18321 [select]:
net/http.(*persistConn).writeLoop(0x10a121e0)
        /usr/local/go/src/net/http/transport.go:1978 +0xb8
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1360 +0x940

goroutine 17984 [IO wait]:
internal/poll.runtime_pollWait(0x65bd2e10, 0x72, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0x107bad84, 0x72, 0x1500, 0x1567, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0x107bad70, 0x10129000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x184
net.(*netFD).Read(0x107bad70, 0x10129000, 0x1567, 0x1567, 0x15f, 0x1bba8, 0x110cd5f0)
        /usr/local/go/src/net/fd_unix.go:202 +0x38
net.(*conn).Read(0x110a6138, 0x10129000, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x5c
crypto/tls.(*atLeastReader).Read(0x110cd5f0, 0x10129000, 0x1567, 0x1567, 0x18384, 0x1f2188, 0xbe1b90)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x78
bytes.(*Buffer).ReadFrom(0xf58d960, 0xbe0c90, 0x110cd5f0, 0x1ffff, 0x1f0b54, 0x912678, 0x9d8128)
        /usr/local/go/src/bytes/buffer.go:207 +0xac
crypto/tls.(*Conn).readFromUntil(0xf58d800, 0xbe1b90, 0x110a6138, 0x5, 0x110a6138, 0x30)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0xf58d800, 0xa38800, 0xf58d8b0, 0x21d7c4)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x108
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xf58d800, 0x1128b000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x140
net/http.(*persistConn).Read(0x10a121e0, 0x1128b000, 0x1000, 0x1000, 0x1, 0x2bb44c, 0x4)
        /usr/local/go/src/net/http/transport.go:1526 +0x1c0
bufio.(*Reader).fill(0x1104dc80)
        /usr/local/go/src/bufio/bufio.go:100 +0x114
bufio.(*Reader).Peek(0x1104dc80, 0x1, 0x18cc16c0, 0x69f269c, 0x0, 0x69f2634, 0x1)
        /usr/local/go/src/bufio/bufio.go:138 +0x38
net/http.(*persistConn).readLoop(0x10a121e0)
        /usr/local/go/src/net/http/transport.go:1679 +0x168
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1359 +0x924

goroutine 18378 [IO wait]:
internal/poll.runtime_pollWait(0x65bd2f9c, 0x72, 0xffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x48
internal/poll.(*pollDesc).wait(0xf7b3dc4, 0x72, 0x1500, 0x1567, 0xffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x80
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xf7b3db0, 0x106bf800, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x184
net.(*netFD).Read(0xf7b3db0, 0x106bf800, 0x1567, 0x1567, 0xca, 0x1bba8, 0x10ecaca0)
        /usr/local/go/src/net/fd_unix.go:202 +0x38
net.(*conn).Read(0x1086b740, 0x106bf800, 0x1567, 0x1567, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x5c
crypto/tls.(*atLeastReader).Read(0x10ecaca0, 0x106bf800, 0x1567, 0x1567, 0x18384, 0x1f2188, 0xbe1b90)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x78
bytes.(*Buffer).ReadFrom(0x102ac760, 0xbe0c90, 0x10ecaca0, 0x1b2fc, 0x1f0b54, 0x912678, 0x9d8128)
        /usr/local/go/src/bytes/buffer.go:207 +0xac
crypto/tls.(*Conn).readFromUntil(0x102ac600, 0xbe1b90, 0x1086b740, 0x5, 0x1086b740, 0x2482000)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xd8
crypto/tls.(*Conn).readRecordOrCCS(0x102ac600, 0xa38800, 0x102ac6b0, 0x2426525)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x108
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0x102ac600, 0x11b95000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x140
net/http.(*persistConn).Read(0x928df40, 0x11b95000, 0x1000, 0x1000, 0x1, 0x2bb44c, 0x4)
        /usr/local/go/src/net/http/transport.go:1526 +0x1c0
bufio.(*Reader).fill(0x11bb8510)
        /usr/local/go/src/bufio/bufio.go:100 +0x114
bufio.(*Reader).Peek(0x11bb8510, 0x1, 0x1c980e00, 0x6ce569c, 0x2484980, 0x1ce73f00, 0x2484940)
        /usr/local/go/src/bufio/bufio.go:138 +0x38
net/http.(*persistConn).readLoop(0x928df40)
        /usr/local/go/src/net/http/transport.go:1679 +0x168
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1359 +0x924

goroutine 18379 [select]:
net/http.(*persistConn).writeLoop(0x928df40)
        /usr/local/go/src/net/http/transport.go:1978 +0xb8
created by net/http.(*Transport).dialConn
        /usr/local/go/src/net/http/transport.go:1360 +0x940
1 Like
#2

Well, that’s pretty self-explanatory, is it not? :wink: :d: is using close to 1.8GB RAM in your case, how much actual RAM+swap does your NAS actually have? If you’re running applications on your device, I’d always recommend to upgrade memory to the max, QNAP has upgradeable memory slots (at least QNAPs I have seen).

Apart from that, you can try the latest git build, not sure if it has any memory optimizations already committed. As a last resort (before splitting up your repos/storages) you can increase swap space, this may help in the short term, but this is not an amazing option for variety of reasons.

#3

1GB Ram
23GB Swap (?)
htop also says 23,4GB swap

free
              total         used         free       shared      buffers
  Mem:      1024360       930944        93416        46416        38028
 Swap:     24542452       461048     24081404
Total:     25566812      1391992     24174820

Not the TS-431+ :frowning:

I only can use compiled versions as compiling on QNAP is painfull / impossible

Also not ideal on QNAP devices

I restarted the NAS and changed the threads from 5 to 2.

#4

I haven’t worked with this specific model, but usually you can change swap into a file on a USB disk, something like this:

swapoff /dev/md4
mdadm -S /dev/md4
mkswap /dev/sds1 -L myswap
swapon /dev/sds1

I am pretty sure it worked on x86-based QNAPs, not sure about arm ones. But this shouldn’t be a problem, you have a ginormous swap as it is (23GB?!), not sure why. At least at the moment you were running free most of the swap was underutilized.

Are you sure you’re not running something else in parallel to :d: that consumes a ton of memory? That would explain behavior you’re seeing assuming :d: reports its memory usage correctly. It shouldn’t be running out of alloc on 1.7GB used with more than 24GB total memory - unless something else consumed 20GB+ of it at that time.

#5

There is not really an application which should use that much memory. Perhaps a faulty qnap app was the culprit and solved with the reboot.

Since the reboot and the reduction to 2 threats it runs without any problems.

#6

If my math is correct, the backup run that failed above was handling around 97GB worth of chunks.

So around 38MB is being used for file buffering plus almost 940MB for the OS and apps, leaving only about 93MB free.

Even if it were possible to increase the swap space, it unfortunately wouldn’t help.

Imagine that you’re making pizza in your kitchen. The number of pizzas you can bake at the same time is determined by the size and number of ovens there are. You can bake a few dozen mini bagel pizzas at a time, a dozen bagel pizzas, a half dozen 6-inch personal pizzas, a couple of 12-inch pizzas, or some combination thereof. With enough counter space, you can swap pizzas in and out of the oven as needed, but what you probably cannot do is bake a 3-foot diameter pizza all at once. It’s analogous to how the combination of memory modules (physical memory) + swap space (virtual memory) works in operating systems.

The amount of physical memory that’s available determines how big a program (code + data) can be run at any given time. Some parts of a program can be swapped out along with any data that’s not required for the particular set of instructions currently being executed, but the instructions plus data cannot exceed the physical memory that’s free.

The QNAP TS-431+ has a 32-bit ARM CPU, so the version of Duplicacy for it is about 27.7MB. With such a relatively compact program, even if 90% of the code could be swapped out, it’d only free up around 25MB (enough to hold roughly 6 chunks of data on average).

A few suggestions…

Multi-threaded programs don’t consume much memory with each additional thread, but the chunks that Duplicacy uploads in parallel does impact memory usage. Decreasing the thread count further down from 2 to 1 might be worthwhile since with 5 threads you were averaging only about 8.2MB/s (66 Mbps), so using 2 threads might not be much faster than with 1 thread.

Next, if you have access to cross compiler tools (or an inexpensive Raspberry Pi), run “strip” on the Duplicacy binary to remove the symbols and other data used when running in a debugger (e.g., for the 64-bit Linux version it shrunk by 25%). While it’s not going to free up hundreds of megabytes, when you’re limited by 1GB of soldered RAM, every byte counts.

Splitting up the repository into smaller repositories is an option, but a simpler technique is to use the -filters option for the set command. Whatever your storage target is, add it multiple times with different storage names and filters. You’re still backing up to the same storage destination, but with different filters to limit what’s being backed up during each run. If for example your repository has three subdirectories (Documents, Music, Videos), your .duplicacy\preferences file could contain the following lines:

"name": "Documents",
"id": "Documents",
"storage": "b2://mybucket",
"filters": "/home/Usefulvid/Documents.txt",

"name": "Music",
"id": "Music",
"storage": "b2://mybucket",
"filters": "/home/Usefulvid/Music.txt",

"name": "Videos",
"id": "Videos",
"storage": "b2://mybucket",
"filters": "/home/Usefulvid/Videos.txt",

In the filter file, /home/Usefulvid/Documents.txt would only include the Documents directory:

+Documents/

(Likewise, same idea for the Music.txt and Videos.txt filter files.)

Then to run the backups:

duplicacy backup -storage Documents
duplicacy backup -storage Music
duplicacy backup -storage Videos

This way you wouldn’t have to set up the Documents, Music and Videos directories as separate repositories.

(10-August-2022: After reviewing what I originally wrote, I noticed that I’d made a mistake on the example snapshot IDs. They should be unique.)

#7

Cute analogy with pizza, but no, that’s not how virtual memory works. VM operates on a page level (typically 4K bytes), it is individual pages that get mapped out either to physical RAM frames or to the swap space. So you can absolutely run programs that require more memory than you physically have RAM, as long as you have enough virtual memory and not failing very large allocs due to memory fragmentation. None of these cases are applicable here. I mean, :d: reports using 1.7GB when there is only 1GB RAM.

The problem with actually using more memory than there is RAM is that VM will constantly swap in and out pages from much slower swap, thus tremendously slowing down execution time. But in many use cases despite allocating a lot of memory, only small part of it is used at any particular time, and swapping out the other parts back to disk has minimal impact on actual execution, that’s why even with large swap utilization it is not always noticeable slower.

Now, if this NAS is indeed 32bit, then a single process cannot allocate more than 2-4GB of virtual memory directly (without memory mapping) no matter how much total memory is available. The example above is close to this limit, but total alloc is not even exceeding 2GB, so I doubt this is a problem at this time.

#8

I haven’t done a deep dive into how ARM’s MMU (Memory Management Unit) works, but on x86 there’s also memory paging at the hardware level. Add in PAE (Physical Address Extension) and things get even more complicated.

While it’s possible to load programs that are larger than the amount of physical memory, any portion of the code and/or data that spills over into swap cannot be executed by the CPU. The code and/or data needs to be swapped back into physical memory in order for the CPU to access it because the CPU doesn’t know anything about file systems (Windows 10/11 use a pair of hidden swap files, C:\pagefile.sys for regular apps and C:\swapfile.sys for Metro/Modern/Universal apps) and swap partitions (as you already know, Un*x systems can mix and match multiple swap files/partitions).

:d: exists at the application layer so it only sees what looks like contiguous memory space even though it’s in reality a combination of physical memory and swap space. The operating system determines which parts of :d: are not required yet and can be moved to swap. Whenever any parts of its code and/or data is required, the OS retrieves it from the swap files/partitions and loads it into main memory where the CPU can reach it and optionally move it into the CPU cache for even faster repeated access.

Using the pizza analogy again, the oven can’t bake an entire 3-foot diameter pie in one pass, but it can easily handle one 1.5-foot long wedge that fits on a typical 17-inch baking sheet. The operator of the oven is responsible for cutting the pie and moving the baking sheet in/out of the oven for cooking each slice because the oven isn’t capable of doing it by itself. The faster the operator works, and the faster the oven can bake, the more :pizza: there is to :yum:.

Very true, for many applications large swap usage might not be noticeable, especially with modern CPUs having triple-level caches (L1, L2, L3) that are also getting larger with each generation – e.g., the 3D chiplet version of the AMD Ryzen 5900X has 192MB of L3 cache.

At least one use case where swapping to virtual memory is very noticeable is when running multiple virtual machines on a host, especially for high density setups when memory ballooning is used by the guests for overprovisioning.

Swapping to a HDD is pretty slow (about 100,000 times slower compared to accessing RAM), but faster and faster SSD/NVMe are helping to narrow the gap. Some day we might be able to replace DRAM with a future super-high-speed flash memory (I wouldn’t mind having a server with 10TB of RAM that also serves as primary storage :grinning:).

According to QNAP, the TS-431+ has a dual-core variant of the ARM Cortex-A15 based on the ARMv7-A core, which in turn is a ARM32 (32-bit) architecture. Interestingly, the ARMv7-A core borrows a memory addressing technique from Intel’s x86 to bump the physical address space to 40-bits (good enough for 1TB of memory).

#9

When using up all available ram on a storage device (let alone spilling into a swap) is touched in the discussion — it’s long as derailed into irrelevant specifics and missed a bug picture: you have to have free unused ram on your nas to get any half-decent performance and responsiveness from your storage device. Using up all that ram evicts disk and/or filesystem cache and performance plummets as disks are very bad at servicing random IO.

But what if your NAS only has 1GB of non-upgradable ram? The solution is painfully obvious: don’t run any user software, let alone memory intensive duplicacy, on it. 1GB is barely enough to provide file services. Barely.

#10

It’s crazy that OEMs heavily market multi-terabyte consumer and SMB grade NAS appliances with budget-friendly hardware specs, then pile on a laundry list of features that include backup services, cloud storage, media streaming, gaming – and not to be outdone by the competition – Docker and/or some kind of hypervisor for virtual machines, often all on a dual-core SoC with 1-2 GB of RAM. While I’m a big fan of squeezing as much as possible out of available hardware, it’s like buying a semitruck powered by a 2-cycle lawnmower engine.

One niche scenario where a NAS with 1GB of RAM would be fine is for archival storage via lightweight network services such as NFS, CIFS, FTP and SFTP (CPUs made within the past 10 years or so have hardware accelerated encryption and compression support). Files are uploaded and just pass thru RAM on its way onto the storage medium so a few simultaneous clients at a time is totally doable.

#11

Ha, I remember running CrashPlan (way worse memory hog than :d:) on a QNAP with 1GB RAM, for quite a while. The real fun is when you run not only out of physical, but also virtual memory, and system starts to kill random processes and/or kernel panic. Debugging this can be fun for the whole family /s

10/10 if you enjoy the pain, otherwise don’t do it :wink:

#12

:scream:

I blame qnap and Synology marketing that market those toys as ”application servers” and replacement for Dropbox, office365, vm server, backup source and target, torrent client, vpn server, Time Machine, media player, media server, and otherwise replacement of the whole industries. 1GB ram? Never mind that, hope users won’t norice and once they bought — what are they are going to do?

#13

Just ordered a Synology DS 920+ :slight_smile:
It has 4GB RAM and I ordered 16GB for the free ram slot.

#14

My strong offtopic advice would be to cancel the order or return it, and buy or build the TrueNAS Core server instead. I cannot stress this enough, I’ve wasted enough time (years!) with Synology, their bugs, their superficial support (along with migration to the online ticket system with impossible to get through first line of support drones; actual engineers are great but they are a victim of a direction the company was taken), their gimmicky approach to features, and their false promises, that everytime I see another fellow happily marching into the same bucket of worms face first I die inside a little.

If you really have to have a Synology (due to some weird bet — can’t think of another reason) — get higher end model. Telltale sign with Synology — internal PSU. It will be slightly more expensive (albeit much better value) but will protect you from quite a chunk of headaches caused by hardware issues. You will still have to deal with crappy Synology software but at least hardware won’t be in the way.

Otherwise — please please strongly reconsider…

1 Like
#15

Officially, J4125 only supports up to 8GB RAM total :wink: It will likely work though, but could be used as a reason to deny support if you will need some.

#16
  1. This was determined to be a marketing limitation, not a technical one; so it indeed will work, albeit with a limited types of ram (but that’s due to decisions Synology made. For example, single-rank ram likely won’t work)
  2. At least in the US (and I will be surprised if EU laws are more lax) warranty and support cannot be denied on that basis. Anecdotally, when I had 32GB in my DS918+ and asked for support due to NVmE hangs rebooting the system where Synology engineering from Taiwan spent a month remotely debugging the issue on my unit, the only conversation regarding ram was “- Did you run the memory test? - Yes. - OK.” That celeron had the same “total maximum system memory 8GB” nonsense in the datasheet.
1 Like
#17

Same error, please fix it.

#18

How much ram is on your system, how many files in the backup set, which version of duplicacy are you using (does it have this fix Memory usage optimization ?) and does DUPLICACY_ATTRIBUTE_THRESHOLD help?

#19

unRAID 6.12.6 with 16 GiB DDR4 Memory

image
image1044Ă—158 13.9 KB

Duplicacy version 3.2.3 (254953)

I’m not sure what the DUPLICACY_ATTRIBUTE_THRESHOLD is, as I couldn’t find it in your wiki. Could you provide more information on this?

And here is the initialization command:

duplicacy init -encrypt -storage-name dva -chunk-size 33554432 -max-chunk-size 67108864 share-photos /mnt/user/backups/duplicacy

And the logs:

Uploaded chunk 1069 size 13797439, 21.60MB/s 00:14:51 65.7%
Uploaded chunk 1070 size 67108864, 21.59MB/s 00:14:48 65.8%
Uploaded chunk 1071 size 9247757, 21.59MB/s 00:14:47 65.8%
runtime: out of memory: cannot allocate 58720256-byte block (3237445632 in use)
fatal error: out of memory

goroutine 18 [running]:
runtime.throw({0x8db0c62, 0xd})
	/usr/local/go/src/runtime/panic.go:1047 +0x4d fp=0x55870a20 sp=0x55870a0c pc=0x80816ed
runtime.(*mcache).allocLarge(0x2aa53088, 0x3618fa7, 0x1)
	/usr/local/go/src/runtime/mcache.go:235 +0x208 fp=0x55870a48 sp=0x55870a20 pc=0x805cb78
runtime.mallocgc(0x3618fa7, 0x8c2d560, 0x1)
	/usr/local/go/src/runtime/malloc.go:1029 +0x458 fp=0x55870a90 sp=0x55870a48 pc=0x8053b48
runtime.makeslice(0x8c2d560, 0x3618fa7, 0x3618fa7)
	/usr/local/go/src/runtime/slice.go:103 +0x4f fp=0x55870aa4 sp=0x55870a90 pc=0x809979f
github.com/bkaradzic/go-lz4.Encode({0x93aa201a, 0x0, 0x7ffffe6}, {0x7dc9c000, 0x35e2e04, 0x8000000})
	/Users/gchen/zincbox/go/pkg/mod/github.com/bkaradzic/go-lz4@v1.0.0/writer.go:112 +0x78 fp=0x55870b4c sp=0x55870aa4 pc=0x845eeb8
github.com/gilbertchen/duplicacy/src.(*Chunk).Encrypt(0x558c8000, {0x5585e800, 0x20, 0x40}, {0x55870d40, 0x20}, 0x0)
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunk.go:311 +0xc4b fp=0x55870d04 sp=0x55870b4c pc=0x8ae1eeb
github.com/gilbertchen/duplicacy/src.(*ChunkOperator).UploadChunk(0x55ad20e0, 0x0, {0x1, {0x559078c0, 0x40}, {0x5a6b0ee0, 0x20}, 0x430, {0x0, 0x0}, ...})
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunkoperator.go:560 +0x57d fp=0x55870df4 sp=0x55870d04 pc=0x8aec8bd
github.com/gilbertchen/duplicacy/src.(*ChunkOperator).Run(0x55ad20e0, 0x0, {0x1, {0x559078c0, 0x40}, {0x5a6b0ee0, 0x20}, 0x430, {0x0, 0x0}, ...})
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunkoperator.go:187 +0x1285 fp=0x55870f34 sp=0x55870df4 pc=0x8aea105
github.com/gilbertchen/duplicacy/src.CreateChunkOperator.func1(0x0)
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunkoperator.go:92 +0x80 fp=0x55870fe8 sp=0x55870f34 pc=0x8ae8680
github.com/gilbertchen/duplicacy/src.CreateChunkOperator.func2()
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunkoperator.go:97 +0x29 fp=0x55870ff0 sp=0x55870fe8 pc=0x8ae85e9
runtime.goexit()
	/usr/local/go/src/runtime/asm_386.s:1326 +0x1 fp=0x55870ff4 sp=0x55870ff0 pc=0x80b46c1
created by github.com/gilbertchen/duplicacy/src.CreateChunkOperator
	/Users/gchen/zincbox/duplicacy/src/duplicacy_chunkoperator.go:87 +0x1ff
#20

Disregard this, in the older versions you cloud have set this environment variable to a low value, e.g. 1 and this would reduce the size of attribute data kept in ram. In the current version this is no longer applicable.

It appears it had allocated slightly over 3GB and ran out of memory.

3GB is very little. Check what is consuming ram on your device.

Even without duplicacy, you want to have a lot of unused ram, to facilitate filesystem caching and maintain server responsivenes.

1 Like