I read through this thread a finally came up with the following setup.
Note that I am using a generic transmission docker image linuxserver/transmission and not something libreELEC specific.
The folder /storage/misc/transmission/config should contain settings.json file with transmission configuration.
In this file I added the port 31777. Note that an rpc password or a non-trivial whitelist must be set. Otherwise we would fail connecting to the server.
Code
LibreELEC:~ # cat /storage/.local/transmission/default
TRANSMISSION_UDP_PORT=31777
TRANSMISSION_TZ=Asia/Jerusalem
LibreELEC:~ # cat /storage/.local/transmission/transmission.service
[Unit]
Description=%p container
Requires=service.system.docker.service
After=service.system.docker.service
[Service]
Restart=always
RestartSec=10s
TimeoutStartSec=0
EnvironmentFile=/storage/.local/transmission/default
ExecStartPre=-/bin/sh -c "mkdir -p /storage/misc/%p/watch /storage/misc/%p/downloads /storage/misc/%p/config"
ExecStart=/bin/sh -c "/storage/.kodi/addons/service.system.docker/bin/docker rm transmission; /storage/.kodi/addons/service.system.docker/bin/docker run \
--rm \
--name=transmission \
--hostname=libreelec-%p \
--volume=/storage/misc/%p/watch:/watch \
--volume=/storage/misc/%p/downloads:/downloads \
--volume=/storage/misc/%p/config:/config \
--publish=9091:9091 \
--publish=${TRANSMISSION_UDP_PORT}:${TRANSMISSION_UDP_PORT} \
--publish=${TRANSMISSION_UDP_PORT}:${TRANSMISSION_UDP_PORT}/udp \
-e TZ=${TRANSMISSION_TZ} \
-e PUID=0 \
-e PGID=0 \
linuxserver/transmission"
ExecStop=/storage/.kodi/addons/service.system.docker/bin/docker stop transmission
[Install]
WantedBy=multi-user.target
LibreELEC:~ # cat /storage/.config/sysctl.d/misc.conf
net.core.wmem_max = 1048576
net.core.rmem_max = 4194304