pbf streets fails with out of memory

[otbutz]

I tried to execute the following command in order to obtain a polylines file for pelias:

go/bin/pbf streets europe.pf > road_network.polylines

and received the following error:

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x7e1342, 0x16)
	/usr/lib/go-1.10/src/runtime/panic.go:619 +0x81
runtime.sysMap(0xdfc5720000, 0x2ca000000, 0x7f5d30d99c00, 0xbe23d8)
	/usr/lib/go-1.10/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0xbc9260, 0x2ca000000, 0xd626680000)
	/usr/lib/go-1.10/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0xbc9260, 0x165000, 0x0)
	/usr/lib/go-1.10/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0xbc9260, 0x165000, 0xbe23e8, 0xbc40b0)
	/usr/lib/go-1.10/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0xbc9260, 0x165000, 0x100, 0xdfc5720000)
	/usr/lib/go-1.10/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
	/usr/lib/go-1.10/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0xbc9260, 0x165000, 0x320010100, 0x7ef708)
	/usr/lib/go-1.10/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x2ca000000, 0xc420a60001, 0x461701)
	/usr/lib/go-1.10/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
	/usr/lib/go-1.10/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0x0)
	/usr/lib/go-1.10/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
	/usr/lib/go-1.10/src/runtime/proc.go:1170

goroutine 56 [running]:
runtime.systemstack_switch()
	/usr/lib/go-1.10/src/runtime/asm_amd64.s:363 fp=0xc420057c20 sp=0xc420057c18 pc=0x45da00
runtime.mallocgc(0x2ca000000, 0x7a3700, 0xc420057d01, 0x43388b)
	/usr/lib/go-1.10/src/runtime/malloc.go:720 +0x8a2 fp=0xc420057cc0 sp=0xc420057c20 pc=0x41b4c2
runtime.newarray(0x7a3700, 0x2200000, 0xc420057d01)
	/usr/lib/go-1.10/src/runtime/malloc.go:855 +0x6a fp=0xc420057cf0 sp=0xc420057cc0 pc=0x41b84a
runtime.makeBucketArray(0x77f780, 0xc420024519, 0xc4207ea198, 0xc4207ea1a0)
	/usr/lib/go-1.10/src/runtime/hashmap.go:881 +0xe2 fp=0xc420057d28 sp=0xc420057cf0 pc=0x413bf2
runtime.hashGrow(0x77f780, 0xd0798e07e0)
	/usr/lib/go-1.10/src/runtime/hashmap.go:905 +0x80 fp=0xc420057d78 sp=0xc420057d28 pc=0x413d70
runtime.mapassign_fast64(0x77f780, 0xd0798e07e0, 0x207563f6, 0xcf5436a0e0)
	/usr/lib/go-1.10/src/runtime/hashmap_fast.go:582 +0x162 fp=0xc420057dc0 sp=0xc420057d78 pc=0x416052
github.com/missinglink/pbf/handler.(*ReadAll).ReadNode(0xd079c79fa0, 0x207563f6, 0x4048d674299d883c, 0x403543549f94855e, 0xdfc56a0090)
	/home/vmuser/go/src/github.com/missinglink/pbf/handler/readall.go:20 +0x5a fp=0xc420057df0 sp=0xc420057dc0 pc=0x676c5a
github.com/missinglink/pbf/proxy.(*WhiteList).ReadNode(0xd0798e0810, 0x207563f6, 0x4048d674299d883c, 0x403543549f94855e, 0xdfc56a0090)
	/home/vmuser/go/src/github.com/missinglink/pbf/proxy/whitelist.go:20 +0x97 fp=0xc420057e28 sp=0xc420057df0 pc=0x6819f7
github.com/missinglink/gosmparse.denseNode(0x818380, 0xd0798e0810, 0xdfa53834a0, 0xdfc22bcb00, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/elements.go:68 +0x144 fp=0xc420057ed8 sp=0xc420057e28 pc=0x5a1fb4
github.com/missinglink/gosmparse.(*Decoder).readElements(0xced96fe690, 0xdfb772a300, 0x1, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:282 +0x639 fp=0xc420057f60 sp=0xc420057ed8 pc=0x5a1579
github.com/missinglink/gosmparse.(*Decoder).Parse.func2(0xd07dce8a80, 0xd07b450d80, 0xced96fe690, 0xd07dce8a70, 0xd4f6c5dec0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:147 +0x9a fp=0xc420057fb8 sp=0xc420057f60 pc=0x5a374a
runtime.goexit()
	/usr/lib/go-1.10/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc420057fc0 sp=0xc420057fb8 pc=0x460591
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:144 +0x235

goroutine 1 [select, 7 minutes]:
github.com/missinglink/gosmparse.(*Decoder).Parse(0xced96fe690, 0x818380, 0xd0798e0810, 0x7a9400, 0x7c4201, 0xd0798e0810)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:162 +0x33e
github.com/missinglink/pbf/parser.(*Parser).Parse(0xc42004b9a0, 0x818380, 0xd0798e0810)
	/home/vmuser/go/src/github.com/missinglink/pbf/parser/parser.go:34 +0x48
github.com/missinglink/pbf/command.parsePBF(0xc4200ca280, 0x7dc8b5, 0x5, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/pbf/command/street_graph.go:343 +0x3d8
github.com/missinglink/pbf/command.StreetGraph(0xc4200ca280, 0x0, 0xc4200ca280)
	/home/vmuser/go/src/github.com/missinglink/pbf/command/street_graph.go:119 +0x15a
github.com/missinglink/pbf/vendor/github.com/codegangsta/cli.HandleAction(0x76ee40, 0x7eef98, 0xc4200ca280, 0xc420066100, 0x0)
	/home/vmuser/go/src/github.com/missinglink/pbf/vendor/github.com/codegangsta/cli/app.go:485 +0xc8
github.com/missinglink/pbf/vendor/github.com/codegangsta/cli.Command.Run(0x7dcfc4, 0x7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7e23ce, 0x19, 0x0, ...)
	/home/vmuser/go/src/github.com/missinglink/pbf/vendor/github.com/codegangsta/cli/command.go:193 +0xa35
github.com/missinglink/pbf/vendor/github.com/codegangsta/cli.(*App).Run(0xc420012b60, 0xc420010090, 0x3, 0x3, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/pbf/vendor/github.com/codegangsta/cli/app.go:250 +0x700
main.main()
	/home/vmuser/go/src/github.com/missinglink/pbf/pbf.go:153 +0x15a5

goroutine 55 [semacquire]:
sync.runtime_SemacquireMutex(0xd07dce8a1c, 0xd4fba7fd00)
	/usr/lib/go-1.10/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xd07dce8a18)
	/usr/lib/go-1.10/src/sync/mutex.go:134 +0x108
github.com/missinglink/pbf/handler.(*ReadAll).ReadNode(0xd079c79fa0, 0x20744b42, 0x4048e69d7342edbc, 0x4035251cc100e6b0, 0xdfc5568990)
	/home/vmuser/go/src/github.com/missinglink/pbf/handler/readall.go:19 +0x32
github.com/missinglink/pbf/proxy.(*WhiteList).ReadNode(0xd0798e0810, 0x20744b42, 0x4048e69d7342edbc, 0x4035251cc100e6b0, 0xdfc5568990)
	/home/vmuser/go/src/github.com/missinglink/pbf/proxy/whitelist.go:20 +0x97
github.com/missinglink/gosmparse.denseNode(0x818380, 0xd0798e0810, 0xdfbb5625a0, 0xdfc17b6800, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/elements.go:68 +0x144
github.com/missinglink/gosmparse.(*Decoder).readElements(0xced96fe690, 0xdfb6d0e000, 0x1, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:282 +0x639
github.com/missinglink/gosmparse.(*Decoder).Parse.func2(0xd07dce8a80, 0xd07b450d80, 0xced96fe690, 0xd07dce8a70, 0xd4f6c5dec0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:147 +0x9a
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:144 +0x235

goroutine 54 [chan send]:
github.com/missinglink/gosmparse.(*Decoder).Parse.func1(0xd07b450d80, 0xced96fe690, 0xd4f6c5dec0, 0xd07dce8a70)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:118 +0x2df
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:105 +0x19c

goroutine 57 [semacquire]:
sync.runtime_SemacquireMutex(0xd07dce8a1c, 0xd4fba7bd01)
	/usr/lib/go-1.10/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xd07dce8a18)
	/usr/lib/go-1.10/src/sync/mutex.go:134 +0x108
github.com/missinglink/pbf/handler.(*ReadAll).ReadNode(0xd079c79fa0, 0x20757467, 0x4048d51cc100e6b0, 0x4035bf9fa97e132c, 0xdfc529bc20)
	/home/vmuser/go/src/github.com/missinglink/pbf/handler/readall.go:19 +0x32
github.com/missinglink/pbf/proxy.(*WhiteList).ReadNode(0xd0798e0810, 0x20757467, 0x4048d51cc100e6b0, 0x4035bf9fa97e132c, 0xdfc529bc20)
	/home/vmuser/go/src/github.com/missinglink/pbf/proxy/whitelist.go:20 +0x97
github.com/missinglink/gosmparse.denseNode(0x818380, 0xd0798e0810, 0xdfbb562660, 0xdfc17b6b00, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/elements.go:68 +0x144
github.com/missinglink/gosmparse.(*Decoder).readElements(0xced96fe690, 0xdfb772a380, 0x1, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:282 +0x639
github.com/missinglink/gosmparse.(*Decoder).Parse.func2(0xd07dce8a80, 0xd07b450d80, 0xced96fe690, 0xd07dce8a70, 0xd4f6c5dec0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:147 +0x9a
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:144 +0x235

goroutine 58 [semacquire]:
sync.runtime_SemacquireMutex(0xd07dce8a1c, 0xd761149d00)
	/usr/lib/go-1.10/src/runtime/sema.go:71 +0x3d
sync.(*Mutex).Lock(0xd07dce8a18)
	/usr/lib/go-1.10/src/sync/mutex.go:134 +0x108
github.com/missinglink/pbf/handler.(*ReadAll).ReadNode(0xd079c79fa0, 0x20750011, 0x4048dc0000000000, 0x4034610624dd2f1b, 0xdfc55689c0)
	/home/vmuser/go/src/github.com/missinglink/pbf/handler/readall.go:19 +0x32
github.com/missinglink/pbf/proxy.(*WhiteList).ReadNode(0xd0798e0810, 0x20750011, 0x4048dc0000000000, 0x4034610624dd2f1b, 0xdfc55689c0)
	/home/vmuser/go/src/github.com/missinglink/pbf/proxy/whitelist.go:20 +0x97
github.com/missinglink/gosmparse.denseNode(0x818380, 0xd0798e0810, 0xdfa5383440, 0xdfc22bc880, 0x0, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/elements.go:68 +0x144
github.com/missinglink/gosmparse.(*Decoder).readElements(0xced96fe690, 0xdfb772a080, 0x1, 0x0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:282 +0x639
github.com/missinglink/gosmparse.(*Decoder).Parse.func2(0xd07dce8a80, 0xd07b450d80, 0xced96fe690, 0xd07dce8a70, 0xd4f6c5dec0)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:147 +0x9a
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:144 +0x235

goroutine 59 [semacquire, 7 minutes]:
sync.runtime_Semacquire(0xd07dce8a8c)
	/usr/lib/go-1.10/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xd07dce8a80)
	/usr/lib/go-1.10/src/sync/waitgroup.go:129 +0x72
github.com/missinglink/gosmparse.(*Decoder).Parse.func3(0xd07dce8a80, 0xd4f6c5df20)
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:159 +0x2b
created by github.com/missinglink/gosmparse.(*Decoder).Parse
	/home/vmuser/go/src/github.com/missinglink/gosmparse/decoder.go:158 +0x297

VM configuration

• Ubuntu 18.04
• 120GB RAM
• 4 cores
• go 1.10
• installed via go get github.com/missinglink/pbf

pbf characteristics:

• europe
• 4.26GB

vmuser@ubuntu1804:~$ go/bin/pbf stats europe.pf
2018/03/05 17:29:37 Nodes: 636905885
2018/03/05 17:29:37 Ways: 107386467
2018/03/05 17:29:37 Relations: 2554467
2018/03/05 17:29:37 NodesWithName: 373730
2018/03/05 17:29:37 WaysWithName: 49843662
2018/03/05 17:29:37 RelationsWithName: 250337
2018/03/05 17:29:37 NodesWithAddress: 63819952
2018/03/05 17:29:37 WaysWithAddress: 0
2018/03/05 17:29:37 RelationsWithAddress: 0
2018/03/05 17:29:37 NodesWithNoTags: 488804679
2018/03/05 17:29:37 WaysWithNoTags: 1215359
2018/03/05 17:29:37 RelationsWithNoTags: 0
2018/03/05 17:29:37 NodesAvgTagCount: 0.21146129299221172
2018/03/05 17:29:37 WaysAvgTagCount: 6.986399234130804
2018/03/05 17:29:37 RelationsAvgTagCount: 2.9026785345518253

[missinglink]

Hi @otbutz

This tool was only designed for small extracts and so hasn't been optimized for/tested on such a large file.

For our production builds of Pelias we still use the scripts written for us by the Valhalla team, you can find a readme here https://github.com/pelias/polylines/wiki/Generating-polylines-from-Valhalla

The Valhalla project merged a bunch of repos in to one at some point and it's possible this is out-of-date, please let me know if you get stuck.

[otbutz]

Thanks for the quick response. I'm will give it a spin and report back.

[otbutz]

I forked the wiki and added my modifications: https://github.com/otbutz/polylines-wiki

[missinglink]

hi @otbutz please see #15

[missinglink]

I also just updated the wiki with the copy you provided in https://github.com/otbutz/polylines-wiki

[otbutz]

Gonna give it a try and will report back!