From 64bf1576759511218bd74ded2793d01425bc913c Mon Sep 17 00:00:00 2001 From: "Silas, Mariusz Grzybacz" Date: Fri, 2 Oct 2026 21:17:41 +0000 Subject: [PATCH 1/3] feat(protocol): add FTP, FTPS and FTPES downloads A new protocol, internal/protocol/ftp, built on github.com/jlaffaye/ftp. - Schemes: ftp:// (plain, or upgraded by the task's "tls" option), ftps:// (implicit TLS) and ftpes:// (explicit TLS, AUTH TLS). - Credentials come from the URL; anonymous otherwise. The password is never put into an error, a log line or a task name. - A file resolves to one file; a directory is walked (MLSD or LIST) to 8 levels and 10,000 files, and a tree past either limit is refused with an error that names it. - Files are split into segments of at least 1 MiB, each fetched on its own login with REST and RETR. A server that refuses a login lowers the task's ceiling to the logins that worked; a server that refuses REST gets one sequential stream per file. Both are saved for resume. - Certificates are verified unless the request's skipVerifyCert is set. A SOCKS5 proxy carries control and data connections; an HTTP proxy cannot carry passive data connections, so FTP connects directly and logs one warning. - The FTP manager is listed before hls and bt, so ftp://host/x.torrent is an FTP file, and auto-extract and auto-torrent accept FTP options for single-file tasks. - controller.Controller gains an optional Logger, set to the downloader's core logger, so fetchers can log. Tests run an in-process ftpserverlib server (test-only dependency). --- go.mod | 4 +- go.sum | 10 + internal/controller/controller.go | 4 + internal/protocol/ftp/conn.go | 399 ++++++++++++ internal/protocol/ftp/download_test.go | 146 +++++ internal/protocol/ftp/fetcher.go | 778 +++++++++++++++++++++++ internal/protocol/ftp/fetcher_manager.go | 106 +++ internal/protocol/ftp/fetcher_test.go | 691 ++++++++++++++++++++ internal/protocol/ftp/testserver_test.go | 385 +++++++++++ internal/protocol/ftp/walk.go | 187 ++++++ pkg/download/downloader.go | 33 +- pkg/download/ftp_test.go | 57 ++ pkg/download/model.go | 5 + pkg/protocol/ftp/model.go | 49 ++ 14 files changed, 2852 insertions(+), 2 deletions(-) create mode 100644 internal/protocol/ftp/conn.go create mode 100644 internal/protocol/ftp/download_test.go create mode 100644 internal/protocol/ftp/fetcher.go create mode 100644 internal/protocol/ftp/fetcher_manager.go create mode 100644 internal/protocol/ftp/fetcher_test.go create mode 100644 internal/protocol/ftp/testserver_test.go create mode 100644 internal/protocol/ftp/walk.go create mode 100644 pkg/download/ftp_test.go create mode 100644 pkg/protocol/ftp/model.go diff --git a/go.mod b/go.mod index 457832c..fcb92d0 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,13 @@ require ( github.com/bodgit/sevenzip v1.6.1 github.com/dop251/goja v0.0.0-20260106131823-651366fbe6e3 github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc + github.com/fclairamb/ftpserverlib v0.31.0 github.com/go-git/go-git/v5 v5.8.1 github.com/google/jsonschema-go v0.4.3 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/imroc/req/v3 v3.52.2 + github.com/jlaffaye/ftp v0.2.4 github.com/matoous/go-nanoid/v2 v2.0.0 github.com/mattn/go-ieproxy v0.0.12 github.com/mholt/archives v0.1.5 @@ -23,6 +25,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 github.com/rs/zerolog v1.31.0 + github.com/spf13/afero v1.15.0 github.com/tetratelabs/wazero v1.12.0 github.com/xiaoqidun/setft v0.0.0-20220310121541-be86327699ad go.etcd.io/bbolt v1.4.3 @@ -59,7 +62,6 @@ require ( github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect - github.com/spf13/afero v1.15.0 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect diff --git a/go.sum b/go.sum index 28babab..aa9972a 100644 --- a/go.sum +++ b/go.sum @@ -205,6 +205,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fclairamb/ftpserverlib v0.31.0 h1:ws4fLJvuU/mgg1uIgRjCJfc6CrFSgIBAOxVsnawXegE= +github.com/fclairamb/ftpserverlib v0.31.0/go.mod h1:C5hUT0sd9P5XsHMxsmpcrT6wdZexu3qtUu1z0Ih6aG0= github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= @@ -347,6 +349,8 @@ github.com/imroc/req/v3 v3.52.2 h1:xJocr1aIv0a2K9knfBQ4JnZHk+kWTITdjf0mgDg229I= github.com/imroc/req/v3 v3.52.2/go.mod h1:dBGsDloOSZJcFs6PnTjZXYBJK70OXbZpizHBLNqcH2k= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jlaffaye/ftp v0.2.4 h1:JqI85DdkfZj8ntaHk8W9U2SC3jNfiPUU70+wtIWmlfE= +github.com/jlaffaye/ftp v0.2.4/go.mod h1:Y1ZnkzxownGIuX7xQ1mQzzkZ21+DbjVIyeKL/V+IIz4= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -527,6 +531,8 @@ github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= @@ -566,6 +572,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA= @@ -612,6 +620,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.1 h1:ASgazW/qBmR+A32MYFDB6E2POoTgOwT509VP0CT/fjs= go.uber.org/mock v0.5.1/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/internal/controller/controller.go b/internal/controller/controller.go index a352a87..5a98a0d 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -3,6 +3,7 @@ package controller import ( "github.com/GopeedLab/gopeed/internal/tempfiles" "github.com/GopeedLab/gopeed/pkg/base" + "github.com/rs/zerolog" "net/http" "net/url" "os" @@ -15,6 +16,9 @@ type Controller struct { TempDir string GetConfig func(v any) GetProxy func(requestProxy *base.RequestProxy) func(*http.Request) (*url.URL, error) + // Logger is the downloader's core logger. It may be nil, in which case + // fetchers log nothing. + Logger *zerolog.Logger FileController //ContextDialer() (proxy.Dialer, error) } diff --git a/internal/protocol/ftp/conn.go b/internal/protocol/ftp/conn.go new file mode 100644 index 0000000..8b56559 --- /dev/null +++ b/internal/protocol/ftp/conn.go @@ -0,0 +1,399 @@ +package ftp + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/textproto" + "net/url" + "strings" + "sync" + "time" + + "github.com/jlaffaye/ftp" + "golang.org/x/net/proxy" + + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +const ( + connectTimeout = 15 * time.Second + readTimeout = 15 * time.Second + quitTimeout = 2 * time.Second +) + +// target is a parsed FTP URL. It holds the password, so it must never be +// formatted into an error or a log line. +type target struct { + host string // host name or IP, the TLS server name + addr string // host:port + user string + pass string + path string // path relative to the login directory, "" for the directory itself + tlsMode string +} + +// parseTarget reads an ftp, ftps or ftpes URL. The path follows RFC 1738: it +// is relative to the login directory, and an encoded leading slash +// (ftp://host/%2Fpub) makes it absolute. +func parseTarget(rawURL, optTLS string) (*target, error) { + u, err := url.Parse(rawURL) + if err != nil { + // url.Error repeats the whole URL, password included. + return nil, errors.New("ftp: invalid URL") + } + t := &target{} + switch strings.ToLower(u.Scheme) { + case "ftps": + t.tlsMode = pftp.TLSImplicit + case "ftpes": + t.tlsMode = pftp.TLSExplicit + case "ftp": + switch strings.ToLower(optTLS) { + case "", pftp.TLSNone: + t.tlsMode = pftp.TLSNone + case pftp.TLSExplicit: + t.tlsMode = pftp.TLSExplicit + case pftp.TLSImplicit: + t.tlsMode = pftp.TLSImplicit + default: + return nil, fmt.Errorf("ftp: unknown tls mode %q", optTLS) + } + default: + return nil, fmt.Errorf("ftp: unsupported scheme %q", u.Scheme) + } + t.host = u.Hostname() + if t.host == "" { + return nil, errors.New("ftp: URL has no host") + } + port := u.Port() + if port == "" { + port = "21" + if t.tlsMode == pftp.TLSImplicit { + port = "990" + } + } + t.addr = net.JoinHostPort(t.host, port) + if u.User != nil { + // Username and Password are already percent-decoded. + t.user = u.User.Username() + t.pass, _ = u.User.Password() + } + if t.user == "" { + t.user, t.pass = "anonymous", "anonymous@" + } + t.path = strings.TrimPrefix(u.Path, "/") + return t, nil +} + +type dialContextFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// session is one logged-in control connection. It owns the network +// connections it opens, so abort can unblock any read or write in progress. +type session struct { + *ftp.ServerConn + + ctrl net.Conn // control connection as dialed, under any TLS layer + dataHost string // where data connections go + + mu sync.Mutex + data net.Conn // current data connection as dialed + closed bool + stop func() bool +} + +// deadline bounds the next control-connection exchange. The control +// connection is idle during a transfer, so every exchange sets a new one. +func (s *session) deadline() { + _ = s.ctrl.SetDeadline(time.Now().Add(readTimeout)) +} + +// clearDeadline lets the control connection stay idle through a transfer. +func (s *session) clearDeadline() { + _ = s.ctrl.SetDeadline(time.Time{}) +} + +// abort closes every connection at once. Pending calls fail immediately. +func (s *session) abort() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + _ = s.ctrl.Close() + if s.data != nil { + _ = s.data.Close() + } +} + +// quit logs out and waits, briefly, for the server to hang up. Servers that +// cap logins count a session until they have seen it end, so the next login +// would otherwise be refused. +func (s *session) quit() { + if s.stop != nil { + s.stop() + } + _ = s.ctrl.SetDeadline(time.Now().Add(quitTimeout)) + if s.ServerConn != nil { + _ = s.ServerConn.Quit() + } + s.abort() +} + +func (s *session) setData(c net.Conn) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + _ = c.Close() + return net.ErrClosed + } + s.data = c + return nil +} + +// ctrlConn is the control connection handed to the ftp library. +type ctrlConn struct { + net.Conn +} + +// RemoteAddr must be a *net.TCPAddr: the library asserts it. A connection +// through a proxy may report something else, and its address is not used +// for data connections anyway (see dialData). +func (c ctrlConn) RemoteAddr() net.Addr { + if a, ok := c.Conn.RemoteAddr().(*net.TCPAddr); ok { + return a + } + return &net.TCPAddr{} +} + +// Close is called by the library's Quit right after it sends QUIT. Reading +// until the server hangs up (bounded by the deadline quit set) tells us the +// server has ended the session. +func (c ctrlConn) Close() error { + _, _ = io.Copy(io.Discard, c.Conn) + return c.Conn.Close() +} + +// dial opens a control connection, upgrades it to TLS when asked, and logs in. +// +// The library is given a dial function for every connection. That is how +// data connections go through the same proxy as the control connection, but +// it also means the library does no TLS on them: with a dial function it +// neither wraps an implicit-TLS control connection nor any data connection. +// We do both here. Explicit TLS (AUTH TLS) is still done by the library, and +// so are PBSZ and PROT P after login. For the same reason the library's +// DialWithTimeout and DialWithContext would have no effect: connectTimeout +// and ctx are applied to our own dials instead. +func (f *Fetcher) dial(ctx context.Context, t *target) (*session, error) { + netDial, proxied, err := f.netDialer(t) + if err != nil { + return nil, err + } + var tlsConf *tls.Config + if t.tlsMode != pftp.TLSNone { + tlsConf = f.tlsConfig(t.host) + } + + dialCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + raw, err := netDial(dialCtx, "tcp", t.addr) + if err != nil { + return nil, err + } + ctrl := raw + if t.tlsMode == pftp.TLSImplicit { + tc := tls.Client(raw, tlsConf) + if err := tc.HandshakeContext(dialCtx); err != nil { + _ = raw.Close() + return nil, err + } + ctrl = tc + } + + s := &session{ctrl: raw, dataHost: t.host} + if a, ok := raw.RemoteAddr().(*net.TCPAddr); ok && !proxied { + // Same IP as the control connection, as the library itself does, + // so a name with several addresses cannot send us elsewhere. + s.dataHost = a.IP.String() + } + s.stop = context.AfterFunc(ctx, s.abort) + s.deadline() + + handedOut := false + opts := []ftp.DialOption{ + ftp.DialWithShutTimeout(readTimeout), + ftp.DialWithDialFunc(func(network, addr string) (net.Conn, error) { + if !handedOut { + // Dial asks for the control connection first, once. + handedOut = true + return ctrlConn{ctrl}, nil + } + return s.dialData(ctx, netDial, tlsConf, addr) + }), + } + switch t.tlsMode { + case pftp.TLSImplicit: + opts = append(opts, ftp.DialWithTLS(tlsConf)) + case pftp.TLSExplicit: + opts = append(opts, ftp.DialWithExplicitTLS(tlsConf)) + } + c, err := ftp.Dial(t.addr, opts...) + if err != nil { + s.stop() + s.abort() + return nil, err + } + s.ServerConn = c + s.deadline() + if err := c.Login(t.user, t.pass); err != nil { + s.stop() + s.abort() + return nil, err + } + return s, nil +} + +// dialData opens a passive data connection. The library takes the host from +// the control connection's peer, which through a proxy is the proxy, so only +// the port is used here. +func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf *tls.Config, addr string) (net.Conn, error) { + _, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + dialCtx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + raw, err := netDial(dialCtx, "tcp", net.JoinHostPort(s.dataHost, port)) + if err != nil { + return nil, err + } + if err := s.setData(raw); err != nil { + return nil, err + } + if tlsConf != nil { + // The handshake runs on the first read, as in the library, because + // some servers only start it once the transfer command is answered. + return tls.Client(raw, tlsConf), nil + } + return raw, nil +} + +// tlsConfig returns the TLS settings for one server. Certificates are +// verified unless the request sets skipVerifyCert, as for HTTP. Data +// connections resume the control connection's TLS session through the shared +// cache, which servers such as vsftpd require by default. +func (f *Fetcher) tlsConfig(host string) *tls.Config { + cfg := &tls.Config{} + if f.manager != nil && f.manager.TLSConfig != nil { + cfg = f.manager.TLSConfig.Clone() + } + cfg.ServerName = host + if f.meta.Req != nil && f.meta.Req.SkipVerifyCert { + cfg.InsecureSkipVerify = true + } + if cfg.ClientSessionCache == nil { + f.sessionCacheOnce.Do(func() { f.sessionCache = tls.NewLRUClientSessionCache(0) }) + cfg.ClientSessionCache = f.sessionCache + } + return cfg +} + +// netDialer picks how to reach the server, from the same proxy settings the +// HTTP protocol reads (the task's proxy, otherwise the global one). +func (f *Fetcher) netDialer(t *target) (dialContextFunc, bool, error) { + direct := (&net.Dialer{Timeout: connectTimeout}).DialContext + pu, err := f.proxyURL(t) + if err != nil { + return nil, false, err + } + if pu == nil { + return direct, false, nil + } + switch strings.ToLower(pu.Scheme) { + case "socks5", "socks5h": + d, err := proxy.FromURL(pu, &net.Dialer{Timeout: connectTimeout}) + if err != nil { + return nil, false, fmt.Errorf("ftp: socks5 proxy: %w", err) + } + if cd, ok := d.(proxy.ContextDialer); ok { + return cd.DialContext, true, nil + } + return func(_ context.Context, network, addr string) (net.Conn, error) { + return d.Dial(network, addr) + }, true, nil + default: + // An HTTP proxy is not used. Every passive data connection goes to + // a port the server picks, and HTTP proxies usually allow CONNECT to + // port 443 only, so the transfer would fail after the login worked. + f.warnOnce.Do(func() { + if l := f.logger(); l != nil { + l.Warn().Msgf("ftp: the %s proxy at %s cannot carry FTP data connections (HTTP proxy), connecting directly; use a SOCKS5 proxy for FTP", strings.ToLower(pu.Scheme), pu.Host) + } + }) + return direct, false, nil + } +} + +func (f *Fetcher) proxyURL(t *target) (*url.URL, error) { + if f.ctl == nil || f.ctl.GetProxy == nil { + return nil, nil + } + handler := f.ctl.GetProxy(f.meta.Req.Proxy) + if handler == nil { + return nil, nil + } + // The proxy chooser reads only the URL; the credentials stay out. + req := &http.Request{URL: &url.URL{Scheme: "ftp", Host: t.addr, Path: "/" + t.path}, Header: http.Header{}} + pu, err := handler(req) + if err != nil { + return nil, errors.New("ftp: cannot read the proxy settings") + } + return pu, nil +} + +// isLoginRefusal tells a server that is full apart from bad credentials. +// 421 always means "not now". 530 means a refused login; it is only a +// capacity limit when these credentials have worked before. The library +// drops the code when USER itself is refused, so a "too many" message counts +// as well. +func isLoginRefusal(err error, credentialsWorked bool) bool { + var te *textproto.Error + if errors.As(err, &te) { + switch te.Code { + case ftp.StatusNotAvailable: + return true + case ftp.StatusNotLoggedIn: + return credentialsWorked || strings.Contains(strings.ToLower(te.Msg), "too many") + } + return false + } + return strings.Contains(strings.ToLower(err.Error()), "too many") +} + +// isPermanent reports errors a retry cannot fix: a 5xx reply and a +// certificate that does not verify. +func isPermanent(err error) bool { + var te *textproto.Error + if errors.As(err, &te) { + return te.Code >= 500 + } + var verifyErr *tls.CertificateVerificationError + if errors.As(err, &verifyErr) { + return true + } + var recordErr tls.RecordHeaderError + if errors.As(err, &recordErr) { + return true + } + return errors.Is(err, errSizeChanged) +} + +func replyCode(err error) int { + var te *textproto.Error + if errors.As(err, &te) { + return te.Code + } + return 0 +} diff --git a/internal/protocol/ftp/download_test.go b/internal/protocol/ftp/download_test.go new file mode 100644 index 0000000..55e26d2 --- /dev/null +++ b/internal/protocol/ftp/download_test.go @@ -0,0 +1,146 @@ +package ftp_test + +import ( + "bytes" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + + "github.com/GopeedLab/gopeed/internal/logger" + iftp "github.com/GopeedLab/gopeed/internal/protocol/ftp" + "github.com/GopeedLab/gopeed/pkg/base" + "github.com/GopeedLab/gopeed/pkg/download" + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +// syncBuffer is a log sink that the downloader's goroutines can share. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// taskEvents records the first done or error event of every task, so the +// test never reads a task's status while the downloader writes it. +type taskEvents struct { + mu sync.Mutex + final map[string]download.EventKey +} + +func (e *taskEvents) listen(ev *download.Event) { + if ev.Key != download.EventKeyDone && ev.Key != download.EventKeyError { + return + } + e.mu.Lock() + defer e.mu.Unlock() + if _, seen := e.final[ev.Task.ID]; !seen { + e.final[ev.Task.ID] = ev.Key + } +} + +func (e *taskEvents) wait(t *testing.T, id string) download.EventKey { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for time.Now().Before(deadline) { + e.mu.Lock() + key, ok := e.final[id] + e.mu.Unlock() + if ok { + return key + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("task %s did not finish", id) + return "" +} + +// TestPasswordNeverLogged runs FTP tasks through the downloader with its core +// logger captured: a success, a missing file and a wrong password, all behind +// an HTTP proxy that FTP must skip with a warning. +func TestPasswordNeverLogged(t *testing.T) { + srv := iftp.StartFTPServer(t, iftp.ServerOptions{User: "u", Pass: "p@ss"}) + want := srv.WriteFile(t, "secret.bin", iftp.RandomBytes(300000, 21)) + + d := download.NewDownloader(&download.DownloaderConfig{Storage: download.NewMemStorage(), StorageDir: t.TempDir()}) + logs := &syncBuffer{} + d.Logger = &logger.Logger{Logger: zerolog.New(logs)} + events := &taskEvents{final: map[string]download.EventKey{}} + d.Listener(events.listen) + if err := d.Setup(); err != nil { + t.Fatal(err) + } + // Close, not Clear: Clear resets the extension list without a lock while + // the last task's done hooks may still read it. + defer d.Close() + cfg, err := d.GetConfig() + if err != nil { + t.Fatal(err) + } + cfg.Proxy = &base.DownloaderProxyConfig{Enable: true, Scheme: "http", Host: "127.0.0.1:1", Usr: "proxyuser", Pwd: "proxysecret"} + if err := d.PutConfig(cfg); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + create := func(rawURL string) string { + id, err := d.CreateDirect(&base.Request{URL: rawURL}, &base.Options{Path: dir, Extra: &pftp.OptsExtra{Connections: 2}}) + if err != nil { + if strings.Contains(err.Error(), "p@ss") || strings.Contains(err.Error(), "p%40ss") { + t.Fatalf("the password leaked into an error: %v", err) + } + t.Fatal(err) + } + return id + } + root := "ftp://u:p%40ss@" + srv.Addr + "/" + + okID := create(root + "secret.bin") + if key := events.wait(t, okID); key != download.EventKeyDone { + t.Fatalf("download ended with %q, logs:\n%s", key, logs.String()) + } + ok := d.GetTask(okID) + if ok.Protocol != "ftp" { + t.Fatalf("protocol %q, want ftp", ok.Protocol) + } + if got := iftp.FileSHA256(t, ok.Meta.SingleFilepath()); got != want { + t.Fatalf("sha256 %s, want %s", got, want) + } + missingID := create(root + "missing.bin") + if key := events.wait(t, missingID); key != download.EventKeyError { + t.Fatalf("a missing file must fail, got %q", key) + } + wrongID := create("ftp://u:wr0ng%40pw@" + srv.Addr + "/secret.bin") + if key := events.wait(t, wrongID); key != download.EventKeyError { + t.Fatalf("a wrong password must fail, got %q", key) + } + missing, wrong := d.GetTask(missingID), d.GetTask(wrongID) + for _, task := range []*download.Task{ok, missing, wrong} { + if name := task.Name(); strings.Contains(name, "p@ss") || strings.Contains(name, "p%40ss") || strings.Contains(name, "wr0ng") { + t.Fatalf("task name %q shows the password", name) + } + } + + out := logs.String() + if !strings.Contains(out, "task id") || !strings.Contains(out, "HTTP proxy") { + t.Fatalf("the captured log misses the expected lines, so it proves nothing:\n%s", out) + } + for _, secret := range []string{"p@ss", "p%40ss", "wr0ng", "proxysecret"} { + if strings.Contains(out, secret) { + t.Fatalf("%q appears in the log:\n%s", secret, out) + } + } +} diff --git a/internal/protocol/ftp/fetcher.go b/internal/protocol/ftp/fetcher.go new file mode 100644 index 0000000..fb4bf6b --- /dev/null +++ b/internal/protocol/ftp/fetcher.go @@ -0,0 +1,778 @@ +// Package ftp downloads files and directory trees over FTP, FTPS (implicit +// TLS) and FTPES (explicit TLS, AUTH TLS). +// +// A file is split into segments of at least 1 MiB. Each segment is fetched on +// its own login with REST and RETR, and its data connection is closed once +// the segment's last byte has arrived. A server that refuses a login lowers +// the task's connection ceiling to the logins that worked, and a server that +// refuses REST gets one sequential stream per file. +package ftp + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "os" + "path" + "sync" + "sync/atomic" + "time" + + "github.com/jlaffaye/ftp" + "github.com/rs/zerolog" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +const ( + // Many servers cap logins per user, so FTP starts lower than HTTP. + defaultConnections = 4 + maxConnections = 256 + minSegmentSize = 1 << 20 + maxRetries = 5 + copyBufferSize = 256 << 10 +) + +// retryDelay is the first wait before a retry. Later retries wait longer. +var retryDelay = 500 * time.Millisecond + +var ( + errSizeChanged = errors.New("ftp: the remote file changed size since the task was resolved") + errShortTransfer = errors.New("ftp: the transfer ended before the segment was complete") + errRestUnsupported = errors.New("ftp: the server does not support REST") + errDiskWrite = errors.New("ftp: cannot write the downloaded data") +) + +type config struct { + Connections int `json:"connections"` +} + +// segment is a byte range of one file. End is inclusive, so an empty file +// has End == Begin-1 and nothing to fetch. +type segment struct { + File int `json:",omitempty"` // index in Res.Files + Begin int64 + End int64 + Downloaded int64 +} + +func (s *segment) remain() int64 { + return s.End - s.Begin + 1 - s.Downloaded +} + +// lane is one worker that logged in, shown as one connection in the stats. +type lane struct { + downloaded int64 + retryTimes int + failed bool + completed bool +} + +// loginError marks a failure to connect or log in, as opposed to a failure +// during a transfer. +type loginError struct{ err error } + +func (e *loginError) Error() string { return e.err.Error() } +func (e *loginError) Unwrap() error { return e.err } + +type Fetcher struct { + manager *FetcherManager + ctl *controller.Controller + config *config + meta *fetcher.FetcherMeta + doneCh chan error + + sessionCacheOnce sync.Once + sessionCache tls.ClientSessionCache + warnOnce sync.Once + // loginOK is set once these credentials have logged in, after which a + // 530 on another login means the server is full. + loginOK atomic.Bool + + // mu guards the download state below. + mu sync.Mutex + segments []*segment + maxConns int + noRest bool + lanes []*lane + done bool + + // runMu serializes Start and Pause. + runMu sync.Mutex + cancel context.CancelFunc + runDone chan struct{} +} + +func (f *Fetcher) Setup(ctl *controller.Controller) { + f.ctl = ctl + f.doneCh = make(chan error, 1) + if f.meta == nil { + f.meta = &fetcher.FetcherMeta{} + } + if ctl != nil && ctl.GetConfig != nil { + ctl.GetConfig(&f.config) + } + if f.config == nil { + f.config = &config{Connections: defaultConnections} + } +} + +func (f *Fetcher) logger() *zerolog.Logger { + if f.ctl == nil { + return nil + } + return f.ctl.Logger +} + +func (f *Fetcher) warn(format string, args ...any) { + if l := f.logger(); l != nil { + l.Warn().Msgf(format, args...) + } +} + +func (f *Fetcher) initOptions() error { + opts := f.meta.Opts + if err := base.ParseOptExtra[pftp.OptsExtra](opts); err != nil { + return err + } + if opts.Extra == nil { + opts.Extra = &pftp.OptsExtra{} + } + return nil +} + +func (f *Fetcher) extra() *pftp.OptsExtra { + if e, ok := f.meta.Opts.Extra.(*pftp.OptsExtra); ok && e != nil { + return e + } + return &pftp.OptsExtra{} +} + +// connections is the number of parallel logins to use. The caller holds mu. +func (f *Fetcher) connections() int { + if f.noRest { + return 1 + } + n := f.extra().Connections + if n <= 0 && f.config != nil { + n = f.config.Connections + } + if n <= 0 { + n = defaultConnections + } + n = min(n, maxConnections) + if f.maxConns > 0 { + n = min(n, f.maxConns) + } + return n +} + +func (f *Fetcher) Resolve(req *base.Request, opts *base.Options) error { + if opts == nil { + opts = &base.Options{} + } + f.meta.Req = req + f.meta.Opts = opts + if err := f.initOptions(); err != nil { + return err + } + t, err := parseTarget(req.URL, f.extra().TLS) + if err != nil { + return err + } + s, err := f.dial(context.Background(), t) + if err != nil { + return err + } + // quit, not abort: a server that caps logins must see this session end + // before the download's own logins arrive. + defer s.quit() + f.loginOK.Store(true) + res, err := f.resolveTarget(s, t) + if err != nil { + return err + } + f.meta.Res = res + f.mu.Lock() + f.segments = nil + f.done = false + f.mu.Unlock() + return nil +} + +// selected returns the indexes of the files to download, in SelectFiles +// order. The caller holds mu or owns the fetcher. +func (f *Fetcher) selected() []int { + if f.meta == nil || f.meta.Res == nil { + return nil + } + if f.meta.Opts != nil && len(f.meta.Opts.SelectFiles) > 0 { + return f.meta.Opts.SelectFiles + } + all := make([]int, len(f.meta.Res.Files)) + for i := range all { + all[i] = i + } + return all +} + +// localPath is where a file lands: under the task path, with the file's +// directories below the URL kept. +func (f *Fetcher) localPath(index int) string { + if f.meta.Res.Name == "" { + return f.meta.SingleFilepath() + } + file := f.meta.Res.Files[index] + return path.Join(f.meta.FolderPath(), file.Path, file.Name) +} + +// remotePath is the server path of a file. A single file uses the URL's own +// path, which a local rename never touches. +func (f *Fetcher) remotePath(t *target, index int) string { + if f.meta.Res.Name == "" { + return t.path + } + file := f.meta.Res.Files[index] + return path.Join(t.path, file.Path, file.Name) +} + +// plan splits every selected file into segments of at least minSegmentSize, +// one per connection at most. The caller holds mu. +func (f *Fetcher) plan() { + conns := f.connections() + f.segments = nil + for _, index := range f.selected() { + size := f.meta.Res.Files[index].Size + n := int64(1) + if !f.noRest { + n = max(1, min(int64(conns), size/minSegmentSize)) + } + for i := int64(0); i < n; i++ { + f.segments = append(f.segments, &segment{ + File: index, + Begin: i * size / n, + End: (i+1)*size/n - 1, + }) + } + } +} + +// planSequential gives every unfinished file one segment from byte 0, +// because without REST a transfer cannot start in the middle. The caller +// holds mu. +func (f *Fetcher) planSequential() { + complete := map[int]bool{} + for _, s := range f.segments { + if _, seen := complete[s.File]; !seen { + complete[s.File] = true + } + if s.remain() > 0 { + complete[s.File] = false + } + } + var out []*segment + for _, index := range f.selected() { + size := f.meta.Res.Files[index].Size + if complete[index] { + out = append(out, &segment{File: index, Begin: 0, End: size - 1, Downloaded: size}) + continue + } + out = append(out, &segment{File: index, Begin: 0, End: size - 1}) + } + f.segments = out + // Bytes counted by the old segments are fetched again. + f.lanes = nil +} + +// openFiles opens every selected file for writing. A fresh plan creates and +// sizes them; a resumed one keeps their content, and a file that has gone +// missing starts over. +func (f *Fetcher) openFiles(fresh bool) (map[int]*os.File, error) { + files := map[int]*os.File{} + closeAll := func() { + for _, file := range files { + _ = file.Close() + } + } + for _, index := range f.selected() { + name := f.localPath(index) + var ( + file *os.File + err error + ) + _, statErr := os.Stat(name) + switch { + case fresh || os.IsNotExist(statErr): + file, err = f.ctl.Touch(name, f.meta.Res.Files[index].Size) + if err == nil && !fresh { + f.mu.Lock() + for _, s := range f.segments { + if s.File == index { + s.Downloaded = 0 + } + } + f.mu.Unlock() + } + case statErr != nil: + err = statErr + default: + file, err = os.OpenFile(name, os.O_RDWR, 0) + } + if err != nil { + closeAll() + return nil, err + } + files[index] = file + } + return files, nil +} + +func (f *Fetcher) Start() error { + f.runMu.Lock() + defer f.runMu.Unlock() + if f.runDone != nil { + select { + case <-f.runDone: + default: + return nil // already running + } + } + if err := f.initOptions(); err != nil { + return err + } + t, err := parseTarget(f.meta.Req.URL, f.extra().TLS) + if err != nil { + return err + } + + // A paused or failed run may have left a result nobody read. + select { + case <-f.doneCh: + default: + } + + f.mu.Lock() + if f.done { + f.mu.Unlock() + f.doneCh <- nil + return nil + } + fresh := len(f.segments) == 0 + if fresh { + f.plan() + } else if f.noRest { + f.planSequential() + } + f.mu.Unlock() + + files, err := f.openFiles(fresh) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(context.Background()) + f.cancel = cancel + f.runDone = make(chan struct{}) + go f.run(ctx, t, files, f.runDone) + return nil +} + +func (f *Fetcher) run(ctx context.Context, t *target, files map[int]*os.File, done chan struct{}) { + defer close(done) + err := f.download(ctx, t, files) + for _, file := range files { + if cerr := file.Close(); cerr != nil && err == nil { + err = cerr + } + } + f.mu.Lock() + if err == nil { + f.done = true + for _, l := range f.lanes { + l.completed = true + } + } + f.mu.Unlock() + if err != nil && ctx.Err() != nil { + return // paused + } + select { + case f.doneCh <- err: + default: + } +} + +func (f *Fetcher) download(ctx context.Context, t *target, files map[int]*os.File) error { + for { + err := f.runWorkers(ctx, t, files) + if !errors.Is(err, errRestUnsupported) { + return err + } + f.mu.Lock() + f.noRest = true + f.planSequential() + f.mu.Unlock() + f.warn("ftp: the server refused REST, so each file is downloaded in one stream") + } +} + +// pool hands out the segments of one run to its workers. +type pool struct { + mu sync.Mutex + queue []*segment + running int // workers still in the run + loggedIn int // workers that logged in during the run +} + +func (p *pool) take() *segment { + p.mu.Lock() + defer p.mu.Unlock() + if len(p.queue) == 0 { + return nil + } + s := p.queue[0] + p.queue = p.queue[1:] + return s +} + +func (p *pool) putBack(s *segment) { + p.mu.Lock() + defer p.mu.Unlock() + p.queue = append([]*segment{s}, p.queue...) +} + +func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.File) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + + f.mu.Lock() + p := &pool{} + for _, s := range f.segments { + if s.remain() > 0 { + p.queue = append(p.queue, s) + } + } + workers := min(f.connections(), len(p.queue)) + f.mu.Unlock() + if workers == 0 { + return nil + } + p.running = workers + + var ( + wg sync.WaitGroup + errMu sync.Mutex + firstErr error + ) + for id := 0; id < workers; id++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + if err := f.worker(runCtx, t, files, p, id); err != nil { + errMu.Lock() + if firstErr == nil { + firstErr = err + } + errMu.Unlock() + cancel() + } + }(id) + } + wg.Wait() + + if err := ctx.Err(); err != nil { + return err + } + if firstErr != nil { + return firstErr + } + f.mu.Lock() + defer f.mu.Unlock() + for _, s := range f.segments { + if s.remain() > 0 { + return errors.New("ftp: the download stopped with data missing") + } + } + return nil +} + +// worker fetches segments from the pool until it is empty. A worker that the +// server turns away before it ever logged in lowers the ceiling and leaves, +// handing its segment to the workers that did log in. +func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, p *pool, id int) error { + var ln *lane + f.mu.Lock() + if id < len(f.lanes) { + ln = f.lanes[id] + } + f.mu.Unlock() + loggedIn := false + onLogin := func() { + if loggedIn { + return + } + loggedIn = true + p.mu.Lock() + p.loggedIn++ + p.mu.Unlock() + f.mu.Lock() + if ln == nil { + ln = &lane{} + f.lanes = append(f.lanes, ln) + } + f.mu.Unlock() + } + + failures := 0 + pendingRetries := 0 + for { + seg := p.take() + if seg == nil { + return nil + } + err := f.fetchSegment(ctx, t, files, seg, &ln, onLogin) + if err == nil { + failures = 0 + continue + } + p.putBack(seg) + if ctx.Err() != nil { + return nil + } + if errors.Is(err, errRestUnsupported) { + return err + } + + var le *loginError + refused := errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) + if refused && !loggedIn { + p.mu.Lock() + ceiling := max(1, p.loggedIn) + leave := p.running > ceiling + if leave { + p.running-- + } + p.mu.Unlock() + if leave { + f.lowerCeiling(ceiling) + return nil + } + } else if !refused && (isPermanent(err) || errors.Is(err, errDiskWrite)) { + f.markFailed(ln) + return err + } + + failures++ + f.mu.Lock() + if ln != nil { + ln.retryTimes += pendingRetries + 1 + pendingRetries = 0 + } else { + pendingRetries++ + } + f.mu.Unlock() + if failures > maxRetries { + f.markFailed(ln) + return fmt.Errorf("ftp: giving up after %d retries: %w", maxRetries, err) + } + select { + case <-ctx.Done(): + return nil + case <-time.After(retryDelay * time.Duration(failures)): + } + } +} + +func (f *Fetcher) lowerCeiling(n int) { + f.mu.Lock() + defer f.mu.Unlock() + if f.maxConns == 0 || n < f.maxConns { + f.maxConns = n + } +} + +func (f *Fetcher) markFailed(ln *lane) { + if ln == nil { + return + } + f.mu.Lock() + ln.failed = true + f.mu.Unlock() +} + +// isRestRefused tells a REST the server does not implement. RETR itself +// fails with 550 and the like, never with these codes. +func isRestRefused(err error) bool { + switch replyCode(err) { + case ftp.StatusBadCommand, ftp.StatusBadArguments, ftp.StatusNotImplemented, ftp.StatusNotImplementedParameter: + return true + } + return false +} + +// fetchSegment logs in, retrieves the segment from its first missing byte +// and writes it in place. It stops reading at the segment's end and closes +// the data connection; the server then reports the transfer aborted, which +// is expected. +func (f *Fetcher) fetchSegment(ctx context.Context, t *target, files map[int]*os.File, seg *segment, ln **lane, onLogin func()) error { + s, err := f.dial(ctx, t) + if err != nil { + return &loginError{err} + } + defer s.quit() + f.loginOK.Store(true) + onLogin() + + remote := f.remotePath(t, seg.File) + f.mu.Lock() + noRest := f.noRest + wantSize := f.meta.Res.Files[seg.File].Size + if noRest { + seg.Downloaded = 0 + } + pos := seg.Begin + seg.Downloaded + end := seg.End + f.mu.Unlock() + + // A file that changed on the server would mix old and new bytes. + s.deadline() + if size, err := s.FileSize(remote); err == nil && size != wantSize { + return errSizeChanged + } else if err != nil && replyCode(err) == 0 { + return err + } + + s.deadline() + var resp *ftp.Response + if noRest { + resp, err = s.Retr(remote) + } else { + resp, err = s.RetrFrom(remote, uint64(pos)) + } + if err != nil { + if !noRest && pos > 0 && isRestRefused(err) { + return errRestUnsupported + } + return err + } + defer resp.Close() + s.clearDeadline() + + file := files[seg.File] + buf := make([]byte, copyBufferSize) + for pos <= end { + n := int(min(int64(len(buf)), end-pos+1)) + _ = resp.SetDeadline(time.Now().Add(readTimeout)) + nr, rerr := resp.Read(buf[:n]) + if nr > 0 { + if _, werr := file.WriteAt(buf[:nr], pos); werr != nil { + return fmt.Errorf("%w: %w", errDiskWrite, werr) + } + pos += int64(nr) + f.mu.Lock() + seg.Downloaded += int64(nr) + if *ln != nil { + (*ln).downloaded += int64(nr) + } + f.mu.Unlock() + } + if rerr != nil { + if pos > end { + break + } + if errors.Is(rerr, io.EOF) { + return errShortTransfer + } + return rerr + } + } + return nil +} + +func (f *Fetcher) Patch(req *base.Request, opts *base.Options) error { + if req == nil { + return nil + } + if req.URL != "" { + f.meta.Req.URL = req.URL + } + if req.Labels != nil { + if f.meta.Req.Labels == nil { + f.meta.Req.Labels = make(map[string]string) + } + for k, v := range req.Labels { + f.meta.Req.Labels[k] = v + } + } + if req.Proxy != nil { + f.meta.Req.Proxy = req.Proxy + } + return nil +} + +func (f *Fetcher) Pause() error { + f.runMu.Lock() + defer f.runMu.Unlock() + if f.cancel != nil { + f.cancel() + } + if f.runDone != nil { + <-f.runDone + } + return nil +} + +func (f *Fetcher) Close() error { + return f.Pause() +} + +func (f *Fetcher) Meta() *fetcher.FetcherMeta { + return f.meta +} + +// Stats reports one connection per worker that logged in, in the HTTP +// protocol's shape. +func (f *Fetcher) Stats() *fetcher.Stats { + f.mu.Lock() + defer f.mu.Unlock() + var total int64 + if n := int64(len(f.lanes)); n > 0 && f.meta != nil && f.meta.Res != nil && f.meta.Res.Size > 0 { + total = (f.meta.Res.Size + n - 1) / n + } + conns := make([]*pftp.StatsConnection, 0, len(f.lanes)) + for _, l := range f.lanes { + conns = append(conns, &pftp.StatsConnection{ + Downloaded: l.downloaded, + Total: total, + Completed: l.completed || (f.done && !l.failed), + Failed: l.failed, + RetryTimes: l.retryTimes, + }) + } + return &fetcher.Stats{Snapshot: &pftp.Stats{Connections: conns}} +} + +// Progress returns the bytes downloaded of each selected file, in +// SelectFiles order. +func (f *Fetcher) Progress() fetcher.Progress { + f.mu.Lock() + defer f.mu.Unlock() + perFile := map[int]int64{} + for _, s := range f.segments { + perFile[s.File] += s.Downloaded + } + sel := f.selected() + p := make(fetcher.Progress, len(sel)) + for i, index := range sel { + p[i] = perFile[index] + } + return p +} + +func (f *Fetcher) Wait() error { + return <-f.doneCh +} diff --git a/internal/protocol/ftp/fetcher_manager.go b/internal/protocol/ftp/fetcher_manager.go new file mode 100644 index 0000000..ef51be2 --- /dev/null +++ b/internal/protocol/ftp/fetcher_manager.go @@ -0,0 +1,106 @@ +package ftp + +import ( + "crypto/tls" + "net/url" + "path" + "strings" + + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +// fetcherData is what a task saves to resume later. Segments has the same +// Begin, End and Downloaded fields every version writes; the other fields are +// optional, so a save without them still restores. +type fetcherData struct { + Segments []*segment + // MaxConns is the login ceiling the server enforced, 0 when none was hit. + MaxConns int `json:",omitempty"` + // NoRest records that the server refused REST, so the task downloads each + // file in one sequential stream. + NoRest bool `json:",omitempty"` +} + +// FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit +// TLS) URLs. +type FetcherManager struct { + // TLSConfig is the base for every TLS connection, for example to trust a + // private root. Nil means the system roots. The task's skipVerifyCert + // still turns verification off. + TLSConfig *tls.Config +} + +func (fm *FetcherManager) Name() string { + return "ftp" +} + +func (fm *FetcherManager) Filters() []*fetcher.SchemeFilter { + return []*fetcher.SchemeFilter{ + {Type: fetcher.FilterTypeUrl, Pattern: "FTP"}, + {Type: fetcher.FilterTypeUrl, Pattern: "FTPS"}, + {Type: fetcher.FilterTypeUrl, Pattern: "FTPES"}, + } +} + +func (fm *FetcherManager) Build() fetcher.Fetcher { + return &Fetcher{manager: fm} +} + +// ParseName names a task before it is resolved. It never includes the +// userinfo of the URL. +func (fm *FetcherManager) ParseName(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "" + } + name := path.Base(strings.TrimSuffix(parsed.Path, "/")) + if name == "" || name == "/" || name == "." { + name = parsed.Hostname() + } + return name +} + +func (fm *FetcherManager) AutoRename() bool { + return true +} + +func (fm *FetcherManager) DefaultConfig() any { + return &config{Connections: defaultConnections} +} + +func (fm *FetcherManager) Store(f fetcher.Fetcher) (any, error) { + ff := f.(*Fetcher) + ff.mu.Lock() + defer ff.mu.Unlock() + segments := make([]*segment, len(ff.segments)) + for i, s := range ff.segments { + c := *s + segments[i] = &c + } + return &fetcherData{ + Segments: segments, + MaxConns: ff.maxConns, + NoRest: ff.noRest, + }, nil +} + +func (fm *FetcherManager) Restore() (v any, f func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher) { + return &fetcherData{}, func(meta *fetcher.FetcherMeta, v any) fetcher.Fetcher { + fd := v.(*fetcherData) + ff := fm.Build().(*Fetcher) + ff.meta = meta + if meta != nil && meta.Opts != nil { + _ = base.ParseOptExtra[pftp.OptsExtra](meta.Opts) + } + ff.segments = fd.Segments + ff.maxConns = fd.MaxConns + ff.noRest = fd.NoRest + return ff + } +} + +func (fm *FetcherManager) Close() error { + return nil +} diff --git a/internal/protocol/ftp/fetcher_test.go b/internal/protocol/ftp/fetcher_test.go new file mode 100644 index 0000000..ba93ea7 --- /dev/null +++ b/internal/protocol/ftp/fetcher_test.go @@ -0,0 +1,691 @@ +package ftp + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/armon/go-socks5" + "github.com/rs/zerolog" + + "github.com/GopeedLab/gopeed/internal/controller" + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +const mib = 1 << 20 + +// testManager trusts the test server's certificate, the way an embedding +// application could add private roots. +func testManager(srv *FTPServer) *FetcherManager { + fm := &FetcherManager{} + if srv != nil { + fm.TLSConfig = &tls.Config{RootCAs: srv.CA} + } + return fm +} + +func testController(fm *FetcherManager) *controller.Controller { + ctl := controller.NewController() + ctl.GetConfig = func(v any) { + data, _ := json.Marshal(fm.DefaultConfig()) + _ = json.Unmarshal(data, v) + } + return ctl +} + +func buildFetcher(fm *FetcherManager) *Fetcher { + f := fm.Build().(*Fetcher) + f.Setup(testController(fm)) + return f +} + +func ftpURL(srv *FTPServer, scheme, userinfo, p string) string { + if userinfo != "" { + userinfo += "@" + } + return scheme + "://" + userinfo + srv.Addr + "/" + strings.TrimPrefix(p, "/") +} + +func resolveTask(t *testing.T, f *Fetcher, rawURL string, extra *pftp.OptsExtra) { + t.Helper() + opts := &base.Options{Path: t.TempDir()} + if extra != nil { + opts.Extra = extra + } + if err := f.Resolve(&base.Request{URL: rawURL}, opts); err != nil { + t.Fatalf("resolve %s: %v", rawURL, err) + } +} + +func waitFetcher(t *testing.T, f fetcher.Fetcher) error { + t.Helper() + done := make(chan error, 1) + go func() { done <- f.Wait() }() + select { + case err := <-done: + return err + case <-time.After(60 * time.Second): + t.Fatal("download did not finish in 60s") + return nil + } +} + +func startAndWait(t *testing.T, f *Fetcher) { + t.Helper() + if err := f.Start(); err != nil { + t.Fatalf("start: %v", err) + } + if err := waitFetcher(t, f); err != nil { + t.Fatalf("download: %v", err) + } +} + +func checkFile(t *testing.T, name, want string) { + t.Helper() + if got := FileSHA256(t, name); got != want { + t.Fatalf("%s: sha256 %s, want %s", name, got, want) + } +} + +func ftpStats(t *testing.T, f *Fetcher) *pftp.Stats { + t.Helper() + stats, ok := f.Stats().Snapshot.(*pftp.Stats) + if !ok { + t.Fatalf("stats snapshot is %T, want *ftp.Stats", f.Stats().Snapshot) + } + return stats +} + +func storeData(t *testing.T, fm *FetcherManager, f *Fetcher) []byte { + t.Helper() + data, err := fm.Store(f) + if err != nil { + t.Fatal(err) + } + blob, err := json.Marshal(data) + if err != nil { + t.Fatal(err) + } + return blob +} + +// restoreFetcher rebuilds a fetcher the way the downloader does after a +// restart: the meta and the save blob both come back from JSON. +func restoreFetcher(t *testing.T, fm *FetcherManager, meta *fetcher.FetcherMeta, blob []byte) *Fetcher { + t.Helper() + metaJSON, err := json.Marshal(meta) + if err != nil { + t.Fatal(err) + } + var restoredMeta fetcher.FetcherMeta + if err := json.Unmarshal(metaJSON, &restoredMeta); err != nil { + t.Fatal(err) + } + v, build := fm.Restore() + if err := json.Unmarshal(blob, v); err != nil { + t.Fatal(err) + } + f := build(&restoredMeta, v).(*Fetcher) + f.Setup(testController(fm)) + return f +} + +func TestResolveFile(t *testing.T) { + for _, noMLST := range []bool{false, true} { + t.Run(fmt.Sprintf("noMLST=%v", noMLST), func(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST}) + data := RandomBytes(100*1024, 1) + srv.WriteFile(t, "pub/file name.bin", data) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/file%20name.bin"), nil) + res := f.Meta().Res + if res.Name != "" { + t.Fatalf("a file must not be a folder resource, got name %q", res.Name) + } + if len(res.Files) != 1 { + t.Fatalf("got %d files, want 1", len(res.Files)) + } + file := res.Files[0] + if file.Name != "file name.bin" || file.Path != "" || file.Size != int64(len(data)) { + t.Fatalf("got file %+v, want name %q, empty path, size %d", file, "file name.bin", len(data)) + } + if res.Size != int64(len(data)) { + t.Fatalf("resource size %d, want %d", res.Size, len(data)) + } + if file.Ctime == nil || time.Since(*file.Ctime) > time.Hour { + t.Fatalf("modification time %v is missing or wrong", file.Ctime) + } + }) + } +} + +func writeTree(t *testing.T, srv *FTPServer) map[string]string { + t.Helper() + return map[string]string{ + "a.txt": srv.WriteFile(t, "pub/tree/a.txt", []byte("hello")), + "sub/b.bin": srv.WriteFile(t, "pub/tree/sub/b.bin", RandomBytes(3*mib+17, 2)), + "sub/deeper/c.dat": srv.WriteFile(t, "pub/tree/sub/deeper/c.dat", RandomBytes(2000, 3)), + "sub/empty.txt": srv.WriteFile(t, "pub/tree/sub/empty.txt", nil), + } +} + +func TestResolveDir(t *testing.T) { + for _, noMLST := range []bool{false, true} { + for _, p := range []string{"pub/tree", "pub/tree/"} { + t.Run(fmt.Sprintf("noMLST=%v/%s", noMLST, p), func(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{NoMLST: noMLST}) + writeTree(t, srv) + if err := os.MkdirAll(filepath.Join(srv.Root, "pub/tree/void"), 0o755); err != nil { + t.Fatal(err) + } + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", p), nil) + res := f.Meta().Res + if res.Name != "tree" { + t.Fatalf("folder name %q, want tree", res.Name) + } + var got []string + for _, file := range res.Files { + got = append(got, fmt.Sprintf("%s|%s|%d", file.Path, file.Name, file.Size)) + } + want := []string{ + "|a.txt|5", + "sub|b.bin|3145745", + "sub|empty.txt|0", + "sub/deeper|c.dat|2000", + } + if !slices.Equal(got, want) { + t.Fatalf("files\n got %q\nwant %q", got, want) + } + if res.Size != 5+3145745+2000 { + t.Fatalf("resource size %d", res.Size) + } + }) + } + } +} + +func TestResolveDirLimits(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + srv.WriteFile(t, "deep/1/2/3/4/5/6/7/8/f.txt", []byte("ok")) + srv.WriteFile(t, "deep9/1/2/3/4/5/6/7/8/9/f.txt", []byte("too deep")) + for i := 0; i < 4; i++ { + srv.WriteFile(t, fmt.Sprintf("many/f%d", i), []byte("x")) + } + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "deep"), nil) + if got := f.Meta().Res.Files[0].Path; got != "1/2/3/4/5/6/7/8" { + t.Fatalf("depth 8 path %q", got) + } + + f = buildFetcher(testManager(srv)) + err := f.Resolve(&base.Request{URL: ftpURL(srv, "ftp", "", "deep9")}, &base.Options{Path: t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "8 levels") { + t.Fatalf("a tree deeper than 8 levels must fail and name the limit, got %v", err) + } + + saved := maxWalkFiles + maxWalkFiles = 3 + defer func() { maxWalkFiles = saved }() + f = buildFetcher(testManager(srv)) + err = f.Resolve(&base.Request{URL: ftpURL(srv, "ftp", "", "many")}, &base.Options{Path: t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "3 files") { + t.Fatalf("a tree over the file cap must fail and name the limit, got %v", err) + } +} + +func TestDownloadPlain(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + data := RandomBytes(3*mib+12345, 4) + want := srv.WriteFile(t, "pub/plain.bin", data) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/plain.bin"), &pftp.OptsExtra{Connections: 4}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + if got := f.Progress().TotalDownloaded(); got != int64(len(data)) { + t.Fatalf("progress %d, want %d", got, len(data)) + } + stats := ftpStats(t, f) + if len(stats.Connections) == 0 || len(stats.Connections) > 4 { + t.Fatalf("got %d connections in stats", len(stats.Connections)) + } + var sum int64 + for _, c := range stats.Connections { + sum += c.Downloaded + if !c.Completed || c.Failed { + t.Fatalf("connection stats %+v after success", c) + } + } + if sum != int64(len(data)) { + t.Fatalf("connections downloaded %d bytes, want %d", sum, len(data)) + } +} + +func TestDownloadDir(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + want := writeTree(t, srv) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "pub/tree"), &pftp.OptsExtra{Connections: 2}) + startAndWait(t, f) + root := f.Meta().FolderPath() + if root != filepath.Join(f.Meta().Opts.Path, "tree") { + t.Fatalf("folder path %q", root) + } + for rel, sum := range want { + checkFile(t, filepath.Join(root, filepath.FromSlash(rel)), sum) + } +} + +func testDownloadTLS(t *testing.T, mode string, schemes map[string]string) { + srv := StartFTPServer(t, ServerOptions{TLS: mode}) + data := RandomBytes(2*mib+999, 5) + want := srv.WriteFile(t, "secure.bin", data) + for scheme, optTLS := range schemes { + t.Run(scheme+"+"+optTLS, func(t *testing.T) { + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, scheme, "", "secure.bin"), &pftp.OptsExtra{Connections: 2, TLS: optTLS}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + }) + } +} + +func TestDownloadExplicitTLS(t *testing.T) { + testDownloadTLS(t, pftp.TLSExplicit, map[string]string{ + "ftpes": "", + "ftp": pftp.TLSExplicit, + }) +} + +func TestDownloadImplicitTLS(t *testing.T) { + testDownloadTLS(t, pftp.TLSImplicit, map[string]string{ + "ftps": "", + "ftp": pftp.TLSImplicit, + }) +} + +func TestBadCertRejectedByDefault(t *testing.T) { + for mode, scheme := range map[string]string{pftp.TLSExplicit: "ftpes", pftp.TLSImplicit: "ftps"} { + t.Run(mode, func(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{TLS: mode}) + want := srv.WriteFile(t, "f.bin", RandomBytes(4096, 6)) + rawURL := ftpURL(srv, scheme, "", "f.bin") + + // No injected roots: the self-signed certificate is unknown. + f := buildFetcher(testManager(nil)) + err := f.Resolve(&base.Request{URL: rawURL}, &base.Options{Path: t.TempDir()}) + var verifyErr *tls.CertificateVerificationError + if !errors.As(err, &verifyErr) { + t.Fatalf("want a certificate verification error, got %v", err) + } + + // The request's existing skipVerifyCert switch accepts it. + f = buildFetcher(testManager(nil)) + if err := f.Resolve(&base.Request{URL: rawURL, SkipVerifyCert: true}, &base.Options{Path: t.TempDir()}); err != nil { + t.Fatalf("skipVerifyCert resolve: %v", err) + } + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + }) + } +} + +func TestSegmentsUseRest(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + srv.ReadDelay.Store(int64(time.Millisecond)) + size := 8 * mib + want := srv.WriteFile(t, "big.bin", RandomBytes(size, 7)) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "big.bin"), &pftp.OptsExtra{Connections: 4}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + rests := srv.RestOffsets() + for _, off := range []int64{2 * mib, 4 * mib, 6 * mib} { + if !slices.Contains(rests, off) { + t.Fatalf("REST offsets %v miss the segment start %d", rests, off) + } + } + for _, off := range rests { + if off <= 0 || off >= int64(size) { + t.Fatalf("REST offset %d out of range", off) + } + } + if got := len(ftpStats(t, f).Connections); got != 4 { + t.Fatalf("stats report %d connections, want 4", got) + } +} + +func TestLoginCapFallsBack(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{MaxLogins: 1}) + want := srv.WriteFile(t, "capped.bin", RandomBytes(4*mib+5, 8)) + + fm := testManager(srv) + f := buildFetcher(fm) + resolveTask(t, f, ftpURL(srv, "ftp", "", "capped.bin"), &pftp.OptsExtra{Connections: 4}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + if srv.Refused() == 0 { + t.Fatal("the server never refused a login, so the cap was not exercised") + } + if got := len(ftpStats(t, f).Connections); got != 1 { + t.Fatalf("stats report %d connections, want 1", got) + } + var saved fetcherData + if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { + t.Fatal(err) + } + if saved.MaxConns != 1 { + t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns) + } +} + +func TestNoRestSequential(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{NoRest: true}) + srv.ReadDelay.Store(int64(time.Millisecond)) + want := srv.WriteFile(t, "norest.bin", RandomBytes(4*mib+3, 9)) + + fm := testManager(srv) + f := buildFetcher(fm) + resolveTask(t, f, ftpURL(srv, "ftp", "", "norest.bin"), &pftp.OptsExtra{Connections: 4}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + if rests := srv.RestOffsets(); len(rests) != 0 { + t.Fatalf("the server cannot have seen a REST, got %v", rests) + } + var saved fetcherData + if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { + t.Fatal(err) + } + if !saved.NoRest { + t.Fatal("the missing REST support was not recorded") + } +} + +// waitProgress waits until the fetcher has between lo and hi bytes. +func waitProgress(t *testing.T, f *Fetcher, lo, hi int64) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + got := f.Progress().TotalDownloaded() + if got >= lo { + if got > hi { + t.Fatalf("progress %d passed %d before the pause; slow the server down", got, hi) + } + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("progress stayed under %d", lo) +} + +func TestResumeAfterKill(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + srv.ReadDelay.Store(int64(3 * time.Millisecond)) + size := int64(4*mib + 777) + want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10)) + + fm := testManager(srv) + f := buildFetcher(fm) + resolveTask(t, f, ftpURL(srv, "ftp", "", "resume.bin"), &pftp.OptsExtra{Connections: 2}) + if err := f.Start(); err != nil { + t.Fatal(err) + } + waitProgress(t, f, size/5, size*4/5) + if err := f.Pause(); err != nil { + t.Fatal(err) + } + blob := storeData(t, fm, f) + + var saved fetcherData + if err := json.Unmarshal(blob, &saved); err != nil { + t.Fatal(err) + } + var savedTotal int64 + resumeAt := map[int64]bool{} + for _, seg := range saved.Segments { + savedTotal += seg.Downloaded + if seg.Begin+seg.Downloaded <= seg.End { + resumeAt[seg.Begin+seg.Downloaded] = true + } + } + if savedTotal == 0 || savedTotal >= size { + t.Fatalf("saved progress %d of %d", savedTotal, size) + } + + before := len(srv.RetrOffsets()) + srv.ReadDelay.Store(0) + restored := restoreFetcher(t, fm, f.Meta(), blob) + if got := restored.Progress().TotalDownloaded(); got != savedTotal { + t.Fatalf("restored progress %d, want %d", got, savedTotal) + } + startAndWait(t, restored) + checkFile(t, restored.Meta().SingleFilepath(), want) + + resumed := srv.RetrOffsets()[before:] + if len(resumed) == 0 { + t.Fatal("no transfer after resume") + } + for _, off := range resumed { + if !resumeAt[off] { + t.Fatalf("resumed RETR at %d, want one of the saved offsets %v", off, resumeAt) + } + } +} + +func TestRetryAfterDroppedConnection(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11)) + srv.DropAfter.Store(mib + 100) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "drop.bin"), &pftp.OptsExtra{Connections: 1}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + offsets := srv.RetrOffsets() + if len(offsets) != 2 || offsets[0] != 0 || offsets[1] <= 0 || offsets[1] > mib+100 { + t.Fatalf("RETR offsets %v, want a restart at the dropped offset", offsets) + } + if got := ftpStats(t, f).Connections[0].RetryTimes; got != 1 { + t.Fatalf("retry count %d, want 1", got) + } +} + +func TestPercentEncodedPassword(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{User: "u", Pass: "p@ss"}) + want := srv.WriteFile(t, "secret.bin", RandomBytes(5000, 12)) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "u:p%40ss", "secret.bin"), nil) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + f = buildFetcher(testManager(srv)) + err := f.Resolve(&base.Request{URL: ftpURL(srv, "ftp", "u:wrong", "secret.bin")}, &base.Options{Path: t.TempDir()}) + if err == nil || !strings.Contains(err.Error(), "530") { + t.Fatalf("a wrong password must fail with 530, got %v", err) + } +} + +func TestOldSaveFormatResumes(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + data := RandomBytes(2*mib+1, 13) + want := srv.WriteFile(t, "old.bin", data) + + fm := testManager(srv) + f := buildFetcher(fm) + resolveTask(t, f, ftpURL(srv, "ftp", "", "old.bin"), &pftp.OptsExtra{Connections: 1}) + + // A save blob that has only the segment fields every version writes. + const done = 700000 + blob := []byte(fmt.Sprintf(`{"Segments":[{"Begin":0,"End":%d,"Downloaded":%d}]}`, len(data)-1, done)) + name := f.Meta().SingleFilepath() + if err := os.WriteFile(name, append(data[:done:done], make([]byte, len(data)-done)...), 0o644); err != nil { + t.Fatal(err) + } + + restored := restoreFetcher(t, fm, f.Meta(), blob) + if got := restored.Progress().TotalDownloaded(); got != done { + t.Fatalf("restored progress %d, want %d", got, done) + } + startAndWait(t, restored) + checkFile(t, name, want) + if got := srv.RetrOffsets(); !slices.Equal(got, []int64{done}) { + t.Fatalf("RETR offsets %v, want [%d]", got, done) + } +} + +func TestDownloadThroughSOCKS5(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSExplicit}) + want := srv.WriteFile(t, "proxied.bin", RandomBytes(2*mib+1, 14)) + + var dials atomic.Int32 + proxySrv, err := socks5.New(&socks5.Config{ + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + dials.Add(1) + var d net.Dialer + return d.DialContext(ctx, network, addr) + }, + }) + if err != nil { + t.Fatal(err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go proxySrv.Serve(ln) + + f := buildFetcher(testManager(srv)) + req := &base.Request{ + URL: ftpURL(srv, "ftpes", "", "proxied.bin"), + Proxy: &base.RequestProxy{ + Mode: base.RequestProxyModeCustom, + Scheme: "socks5", + Host: ln.Addr().String(), + }, + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil { + t.Fatal(err) + } + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + // Resolve's control and listing, then per worker one control and one + // data connection at least: every one of them went through the proxy. + if got := dials.Load(); got < 4 { + t.Fatalf("the proxy saw %d connections, want every control and data connection", got) + } +} + +func TestHTTPProxyFallsBackToDirect(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + want := srv.WriteFile(t, "direct.bin", RandomBytes(100000, 15)) + + var logs bytes.Buffer + logger := zerolog.New(&logs) + fm := testManager(srv) + f := fm.Build().(*Fetcher) + ctl := testController(fm) + ctl.Logger = &logger + f.Setup(ctl) + req := &base.Request{ + URL: ftpURL(srv, "ftp", "", "direct.bin"), + Proxy: &base.RequestProxy{ + Mode: base.RequestProxyModeCustom, + Scheme: "http", + Host: "127.0.0.1:1", + Usr: "proxyuser", + Pwd: "proxysecret", + }, + } + if err := f.Resolve(req, &base.Options{Path: t.TempDir(), Extra: &pftp.OptsExtra{Connections: 2}}); err != nil { + t.Fatal(err) + } + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + + out := logs.String() + if n := strings.Count(out, "HTTP proxy"); n != 1 { + t.Fatalf("want exactly one HTTP proxy warning, got %d in %q", n, out) + } + if strings.Contains(out, "proxysecret") { + t.Fatalf("the proxy password leaked into the log: %q", out) + } +} + +func TestTorrentNameIsFile(t *testing.T) { + fm := testManager(nil) + for _, u := range []string{"ftp://h/x.torrent", "ftps://h/x.torrent", "ftpes://h/x.torrent", "FTP://h/v.m3u8"} { + matched := false + for _, filter := range fm.Filters() { + if filter.Match(u) { + matched = true + } + } + if !matched { + t.Fatalf("%s is not matched by the FTP filters", u) + } + } + if fm.Name() != "ftp" { + t.Fatalf("manager name %q", fm.Name()) + } + + srv := StartFTPServer(t, ServerOptions{}) + want := srv.WriteFile(t, "x.torrent", []byte("d8:announce0:e")) + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "x.torrent"), nil) + res := f.Meta().Res + if res.Name != "" || len(res.Files) != 1 || res.Files[0].Name != "x.torrent" { + t.Fatalf("x.torrent must resolve to one FTP file, got name %q and %d files", res.Name, len(res.Files)) + } + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) +} + +func TestParseNameHidesCredentials(t *testing.T) { + fm := testManager(nil) + for u, want := range map[string]string{ + "ftp://u:p%40ss@example.com/pub/a.iso": "a.iso", + "ftp://u:p%40ss@example.com/": "example.com", + "ftps://u:p%40ss@example.com": "example.com", + "ftp://u:p%40ss@example.com/pub/dir/": "dir", + } { + if got := fm.ParseName(u); got != want { + t.Fatalf("ParseName(%q) = %q, want %q", u, got, want) + } + } +} + +func TestUnsafeEntryNames(t *testing.T) { + for name, ok := range map[string]bool{ + "a.txt": true, "..a": true, "a b": true, + "": false, ".": false, "..": false, "a/b": false, `a\b`: false, "../x": false, "x\x00y": false, + } { + if got := safeEntryName(name); got != ok { + t.Fatalf("safeEntryName(%q) = %v, want %v", name, got, ok) + } + } +} diff --git a/internal/protocol/ftp/testserver_test.go b/internal/protocol/ftp/testserver_test.go new file mode 100644 index 0000000..d26225a --- /dev/null +++ b/internal/protocol/ftp/testserver_test.go @@ -0,0 +1,385 @@ +package ftp + +import ( + "bufio" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "errors" + "io" + "math/big" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + ftpserver "github.com/fclairamb/ftpserverlib" + "github.com/spf13/afero" + + pftp "github.com/GopeedLab/gopeed/pkg/protocol/ftp" +) + +// ServerOptions are the knobs of the in-process FTP test server. +type ServerOptions struct { + // TLS is pftp.TLSNone (or empty), pftp.TLSExplicit or pftp.TLSImplicit. + // Explicit mode refuses plain logins, so a passing test proves TLS was used. + TLS string + // MaxLogins caps concurrent control connections. Beyond it the server + // greets with "421 Too many connections" and hangs up. Zero means no cap. + MaxLogins int + // NoRest answers "502" to REST. It filters the plain control stream, so it + // works with TLSNone and TLSImplicit only. + NoRest bool + // NoMLST disables MLST and MLSD, so clients fall back to SIZE and LIST. + NoMLST bool + // User and Pass are the only accepted credentials. An empty User accepts + // any login, anonymous included. + User string + Pass string +} + +// FTPServer is an ftpserverlib server over a temp directory. +type FTPServer struct { + Addr string // host:port + Root string // directory served as "/" + // CA trusts the server's self-signed certificate. + CA *x509.CertPool + + opts ServerOptions + tlsConf *tls.Config + server *ftpserver.FtpServer + + // ReadDelay is slept before every read of a transferred file, in + // nanoseconds, to slow downloads down. Tests may change it at any time. + ReadDelay atomic.Int64 + + // DropAfter, when positive, makes the next transfer fail once after it + // has sent that many bytes, as if the data connection were killed. + DropAfter atomic.Int64 + + mu sync.Mutex + active int + retrs []int64 // offset of every RETR, 0 when no REST came first + refused int + logins int +} + +// StartFTPServer starts a server and stops it when the test ends. +func StartFTPServer(t *testing.T, opts ServerOptions) *FTPServer { + t.Helper() + s := &FTPServer{Root: t.TempDir(), opts: opts} + s.tlsConf, s.CA = selfSignedTLS(t) + + base, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var ln net.Listener = base + if opts.TLS == pftp.TLSImplicit { + ln = tls.NewListener(ln, s.tlsConf) + } + ln = &limitListener{Listener: ln, srv: s} + + s.Addr = base.Addr().String() + s.server = ftpserver.NewFtpServer(&mainDriver{srv: s, listener: ln}) + if err := s.server.Listen(); err != nil { + t.Fatal(err) + } + go s.server.Serve() + t.Cleanup(func() { _ = s.server.Stop() }) + return s +} + +// WriteFile creates a file under the served root and returns its SHA-256. +func (s *FTPServer) WriteFile(t *testing.T, rel string, data []byte) string { + t.Helper() + name := filepath.Join(s.Root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, data, 0o644); err != nil { + t.Fatal(err) + } + return SHA256Hex(data) +} + +// RetrOffsets returns the start offset of every RETR the server received. +func (s *FTPServer) RetrOffsets() []int64 { + s.mu.Lock() + defer s.mu.Unlock() + return append([]int64(nil), s.retrs...) +} + +// RestOffsets returns the non-zero RETR offsets, which only a REST sets. +func (s *FTPServer) RestOffsets() []int64 { + var out []int64 + for _, off := range s.RetrOffsets() { + if off != 0 { + out = append(out, off) + } + } + return out +} + +// Refused returns how many connections were turned away with 421. +func (s *FTPServer) Refused() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.refused +} + +// Logins returns the number of successful logins. +func (s *FTPServer) Logins() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.logins +} + +func (s *FTPServer) acquire() bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.opts.MaxLogins > 0 && s.active >= s.opts.MaxLogins { + s.refused++ + return false + } + s.active++ + return true +} + +func (s *FTPServer) release() { + s.mu.Lock() + s.active-- + s.mu.Unlock() +} + +// limitListener counts control connections and refuses those beyond +// MaxLogins with a 421 greeting, like vsftpd's max_clients and max_per_ip. +type limitListener struct { + net.Listener + srv *FTPServer +} + +func (l *limitListener) Accept() (net.Conn, error) { + for { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + if !l.srv.acquire() { + go func() { + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = io.WriteString(c, "421 Too many connections\r\n") + _ = c.Close() + }() + continue + } + var conn net.Conn = &countedConn{Conn: c, release: l.srv.release} + if l.srv.opts.NoRest { + conn = &noRestConn{countedConn: conn.(*countedConn), in: bufio.NewReader(c)} + } + return conn, nil + } +} + +type countedConn struct { + net.Conn + once sync.Once + release func() +} + +func (c *countedConn) Close() error { + c.once.Do(c.release) + return c.Conn.Close() +} + +// noRestConn answers REST itself with 502 and hides it from the server. +type noRestConn struct { + *countedConn + in *bufio.Reader + pending []byte +} + +func (c *noRestConn) Read(p []byte) (int, error) { + for len(c.pending) == 0 { + line, err := c.in.ReadBytes('\n') + if len(line) > 0 { + if strings.HasPrefix(strings.ToUpper(string(line)), "REST ") { + if _, werr := io.WriteString(c.Conn, "502 REST not implemented\r\n"); werr != nil { + return 0, werr + } + } else { + c.pending = line + } + } + if err != nil { + if len(c.pending) > 0 { + break + } + return 0, err + } + } + n := copy(p, c.pending) + c.pending = c.pending[n:] + return n, nil +} + +type mainDriver struct { + srv *FTPServer + listener net.Listener +} + +func (d *mainDriver) GetSettings() (*ftpserver.Settings, error) { + settings := &ftpserver.Settings{ + Listener: d.listener, + DisableMLSD: d.srv.opts.NoMLST, + DisableMLST: d.srv.opts.NoMLST, + } + switch d.srv.opts.TLS { + case pftp.TLSExplicit: + settings.TLSRequired = ftpserver.MandatoryEncryption + case pftp.TLSImplicit: + settings.TLSRequired = ftpserver.ImplicitEncryption + } + return settings, nil +} + +func (d *mainDriver) ClientConnected(ftpserver.ClientContext) (string, error) { + return "gopeed test server", nil +} + +func (d *mainDriver) ClientDisconnected(ftpserver.ClientContext) {} + +func (d *mainDriver) AuthUser(_ ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + if d.srv.opts.User != "" && (user != d.srv.opts.User || pass != d.srv.opts.Pass) { + return nil, errors.New("bad credentials") + } + d.srv.mu.Lock() + d.srv.logins++ + d.srv.mu.Unlock() + return &clientDriver{Fs: afero.NewBasePathFs(afero.NewOsFs(), d.srv.Root), srv: d.srv}, nil +} + +func (d *mainDriver) GetTLSConfig() (*tls.Config, error) { + return d.srv.tlsConf, nil +} + +type clientDriver struct { + afero.Fs + srv *FTPServer +} + +// GetHandle sees the REST offset of every transfer. The server seeks the +// returned file to that offset itself. +func (d *clientDriver) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { + f, err := d.Fs.OpenFile(name, flags, 0o644) + if err != nil { + return nil, err + } + if flags == os.O_RDONLY { + d.srv.mu.Lock() + d.srv.retrs = append(d.srv.retrs, offset) + d.srv.mu.Unlock() + } + return &slowFile{File: f, srv: d.srv, dropAfter: d.srv.DropAfter.Swap(0)}, nil +} + +var errDropped = errors.New("test server dropped the transfer") + +type slowFile struct { + afero.File + srv *FTPServer + sent int64 + dropAfter int64 +} + +func (f *slowFile) Read(p []byte) (int, error) { + if delay := time.Duration(f.srv.ReadDelay.Load()); delay > 0 { + time.Sleep(delay) + if len(p) > 16<<10 { + p = p[:16<<10] + } + } + if f.dropAfter > 0 { + if f.sent >= f.dropAfter { + return 0, errDropped + } + if rest := f.dropAfter - f.sent; int64(len(p)) > rest { + p = p[:rest] + } + } + n, err := f.File.Read(p) + f.sent += int64(n) + return n, err +} + +// selfSignedTLS returns a server config with a fresh certificate for +// 127.0.0.1 and localhost, and a pool that trusts it. +func selfSignedTLS(t *testing.T) (*tls.Config, *x509.CertPool) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: "gopeed ftp test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(cert) + return &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key, Leaf: cert}}, + MinVersion: tls.VersionTLS12, + }, pool +} + +func SHA256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func FileSHA256(t *testing.T, name string) string { + t.Helper() + data, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + return SHA256Hex(data) +} + +// RandomBytes returns n reproducible pseudo-random bytes. +func RandomBytes(n int, seed byte) []byte { + out := make([]byte, n) + x := uint32(seed) + 1 + for i := range out { + x ^= x << 13 + x ^= x >> 17 + x ^= x << 5 + out[i] = byte(x) + } + return out +} diff --git a/internal/protocol/ftp/walk.go b/internal/protocol/ftp/walk.go new file mode 100644 index 0000000..d75215e --- /dev/null +++ b/internal/protocol/ftp/walk.go @@ -0,0 +1,187 @@ +package ftp + +import ( + "cmp" + "fmt" + "path" + "slices" + "strings" + "time" + + "github.com/jlaffaye/ftp" + + "github.com/GopeedLab/gopeed/pkg/base" +) + +// Limits of a directory download. A tree past either one is refused with an +// error that names the limit; it is never cut short. +const maxWalkDepth = 8 + +// maxWalkFiles is a variable so tests can lower it. +var maxWalkFiles = 10000 + +// resolveTarget describes the URL's path: one file, or a directory walked +// into a multi-file resource whose paths are relative to it. +func (f *Fetcher) resolveTarget(s *session, t *target) (*base.Resource, error) { + isDir, size, mtime, err := f.stat(s, t.path) + if err != nil { + return nil, err + } + if !isDir { + return &base.Resource{ + Size: size, + Range: true, + Files: []*base.FileInfo{{ + Name: path.Base(t.path), + Size: size, + Ctime: mtime, + }}, + }, nil + } + + files, err := f.walk(s, t.path) + if err != nil { + return nil, err + } + name := path.Base(strings.TrimSuffix(t.path, "/")) + if name == "" || name == "." || name == "/" { + name = t.host + } + res := &base.Resource{Name: name, Range: true, Files: files} + res.CalcSize(nil) + return res, nil +} + +// stat tells a file from a directory. MLST answers both at once. Without +// it, SIZE answers for a file, and a directory is one we can change into. +func (f *Fetcher) stat(s *session, p string) (isDir bool, size int64, mtime *time.Time, err error) { + if p == "" || strings.HasSuffix(p, "/") { + return true, 0, nil, nil + } + s.deadline() + if entry, mlstErr := s.GetEntry(p); mlstErr == nil { + switch entry.Type { + case ftp.EntryTypeFolder: + return true, 0, nil, nil + case ftp.EntryTypeFile: + if !entry.Time.IsZero() { + t := entry.Time + mtime = &t + } + return false, int64(entry.Size), mtime, nil + } + // A link: SIZE below follows it on the server. + } + // MLST is missing or refused the path: the older commands decide. A + // broken connection fails them too, with its own error. + + s.deadline() + size, sizeErr := s.FileSize(p) + if sizeErr == nil { + if s.IsGetTimeSupported() { + s.deadline() + if t, err := s.GetTime(p); err == nil { + mtime = &t + } + } + return false, size, mtime, nil + } + if replyCode(sizeErr) == 0 { + return false, 0, nil, sizeErr + } + + // SIZE failed: a directory, or a missing path. + s.deadline() + cwd, err := s.CurrentDir() + if err != nil { + return false, 0, nil, err + } + s.deadline() + if err := s.ChangeDir(p); err != nil { + return false, 0, nil, sizeErr + } + s.deadline() + if err := s.ChangeDir(cwd); err != nil { + return false, 0, nil, err + } + return true, 0, nil, nil +} + +// safeEntryName reports whether a name from a listing can become a local +// path element. A hostile server could otherwise list "../x" and write +// outside the task directory. +func safeEntryName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + return !strings.ContainsAny(name, "/\\\x00") +} + +// walk lists a directory tree, at most maxWalkDepth levels below root and +// maxWalkFiles files. Files come back sorted by path, then name. +func (f *Fetcher) walk(s *session, root string) ([]*base.FileInfo, error) { + type dir struct { + remote string // path on the server + rel string // path below root, "" for root itself + depth int + } + var files []*base.FileInfo + stack := []dir{{remote: strings.TrimSuffix(root, "/"), rel: "", depth: 0}} + for len(stack) > 0 { + d := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + s.deadline() + entries, err := s.List(d.remote) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Name == "." || e.Name == ".." { + continue + } + if !safeEntryName(e.Name) { + f.warn("ftp: skipping a listed name that is not a safe file name: %q", e.Name) + continue + } + remote := path.Join(d.remote, e.Name) + rel := path.Join(d.rel, e.Name) + if d.remote == "" { + remote = e.Name + } + entryType, size := e.Type, int64(e.Size) + if entryType == ftp.EntryTypeLink { + // A link to a file has a size; a link to a directory is + // not followed, so a loop cannot make the walk endless. + s.deadline() + linkSize, err := s.FileSize(remote) + if err != nil { + f.warn("ftp: skipping link %q: %v", rel, err) + continue + } + entryType, size = ftp.EntryTypeFile, linkSize + } + switch entryType { + case ftp.EntryTypeFolder: + if d.depth+1 > maxWalkDepth { + return nil, fmt.Errorf("ftp: the folder is deeper than %d levels, the limit for a folder download", maxWalkDepth) + } + stack = append(stack, dir{remote: remote, rel: rel, depth: d.depth + 1}) + case ftp.EntryTypeFile: + if len(files) >= maxWalkFiles { + return nil, fmt.Errorf("ftp: the folder has more than %d files, the limit for a folder download", maxWalkFiles) + } + fi := &base.FileInfo{Name: e.Name, Path: d.rel, Size: size} + if !e.Time.IsZero() { + t := e.Time + fi.Ctime = &t + } + files = append(files, fi) + } + } + } + slices.SortFunc(files, func(a, b *base.FileInfo) int { + return cmp.Or(strings.Compare(a.Path, b.Path), strings.Compare(a.Name, b.Name)) + }) + return files, nil +} diff --git a/pkg/download/downloader.go b/pkg/download/downloader.go index 2ab39c7..00be30a 100644 --- a/pkg/download/downloader.go +++ b/pkg/download/downloader.go @@ -24,6 +24,7 @@ import ( "github.com/GopeedLab/gopeed/internal/tempfiles" webviewproxy "github.com/GopeedLab/gopeed/internal/webview/proxy" "github.com/GopeedLab/gopeed/pkg/base" + "github.com/GopeedLab/gopeed/pkg/protocol/ftp" "github.com/GopeedLab/gopeed/pkg/protocol/http" "github.com/GopeedLab/gopeed/pkg/util" gonanoid "github.com/matoous/go-nanoid/v2" @@ -427,6 +428,9 @@ func (d *Downloader) setupFetcher(fm fetcher.FetcherManager, fetcher fetcher.Fet ctl.GetConfig = func(v any) { d.getProtocolConfig(fm.Name(), v) } + if d.Logger != nil { + ctl.Logger = &d.Logger.Logger + } // Get proxy config, task request proxy config has higher priority, then use global proxy config ctl.GetProxy = func(requestProxy *base.RequestProxy) func(*gohttp.Request) (*url.URL, error) { // Registry-backed blob URLs are loopback capability URLs and must never be @@ -1478,7 +1482,7 @@ func (d *Downloader) watch(task *Task) { d.triggerWebhooks(WebhookEventDownloadDone, task, nil) d.triggerScripts(ScriptEventDownloadDone, task, nil) - if e, ok := task.Meta.Opts.Extra.(*http.OptsExtra); ok { + if e := postDownloadExtra(task.Meta); e != nil { downloadFilePath := task.Meta.SingleFilepath() cfg, _ := d.GetConfig() @@ -1538,6 +1542,33 @@ func (d *Downloader) watch(task *Task) { } } +// postDownloadExtra returns the auto-torrent and auto-extract options of a +// finished single-file task, in the HTTP options shape the extraction code +// takes. FTP options carry the same fields. An FTP folder has no single file +// to act on, so it gets nil, as does any other protocol. +func postDownloadExtra(meta *fetcher.FetcherMeta) *http.OptsExtra { + if meta == nil || meta.Opts == nil { + return nil + } + switch e := meta.Opts.Extra.(type) { + case *http.OptsExtra: + return e + case *ftp.OptsExtra: + if meta.Res == nil || meta.Res.Name != "" || len(meta.Res.Files) != 1 { + return nil + } + return &http.OptsExtra{ + Connections: e.Connections, + AutoTorrent: e.AutoTorrent, + DeleteTorrentAfterDownload: e.DeleteTorrentAfterDownload, + AutoExtract: e.AutoExtract, + ArchivePassword: e.ArchivePassword, + DeleteAfterExtract: e.DeleteAfterExtract, + } + } + return nil +} + func (d *Downloader) doOnError(task *Task, err error) { d.handleOnError(task, err, false) } diff --git a/pkg/download/ftp_test.go b/pkg/download/ftp_test.go new file mode 100644 index 0000000..f292dd7 --- /dev/null +++ b/pkg/download/ftp_test.go @@ -0,0 +1,57 @@ +package download + +import ( + "testing" + + "github.com/GopeedLab/gopeed/internal/fetcher" + "github.com/GopeedLab/gopeed/pkg/base" + "github.com/GopeedLab/gopeed/pkg/protocol/ftp" + "github.com/GopeedLab/gopeed/pkg/protocol/http" +) + +// TestTorrentNameIsFile checks the default manager order: an FTP URL whose +// name looks like a torrent or a playlist is still an FTP download. +func TestTorrentNameIsFile(t *testing.T) { + d := NewDownloader(nil) + for u, want := range map[string]string{ + "ftp://h/x.torrent": "ftp", + "ftp://h/v.m3u8": "ftp", + "ftps://h/x.torrent": "ftp", + "ftpes://h/a.bin": "ftp", + "https://h/v.m3u8": "hls", + "https://h/x.torrent": "http", + "magnet:?xt=urn:btih:": "bt", + } { + fm, err := d.parseFm(u) + if err != nil { + t.Fatalf("%s: %v", u, err) + } + if fm.Name() != want { + t.Fatalf("%s goes to %q, want %q", u, fm.Name(), want) + } + } +} + +func TestPostDownloadExtraAcceptsFTP(t *testing.T) { + yes := true + file := &base.Resource{Files: []*base.FileInfo{{Name: "a.zip"}}} + folder := &base.Resource{Name: "dir", Files: []*base.FileInfo{{Name: "a.zip"}}} + + httpExtra := &http.OptsExtra{AutoExtract: &yes} + if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: httpExtra}}); got != httpExtra { + t.Fatalf("HTTP options must pass through unchanged, got %+v", got) + } + + ftpExtra := &ftp.OptsExtra{Connections: 4, AutoTorrent: &yes, DeleteTorrentAfterDownload: &yes, AutoExtract: &yes, ArchivePassword: "pw", DeleteAfterExtract: true} + got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{Extra: ftpExtra}}) + if got == nil || got.AutoTorrent != &yes || got.DeleteTorrentAfterDownload != &yes || got.AutoExtract != &yes || + got.ArchivePassword != "pw" || !got.DeleteAfterExtract { + t.Fatalf("FTP options were not carried over: %+v", got) + } + if got := postDownloadExtra(&fetcher.FetcherMeta{Res: folder, Opts: &base.Options{Extra: ftpExtra}}); got != nil { + t.Fatalf("an FTP folder has no single file to extract, got %+v", got) + } + if got := postDownloadExtra(&fetcher.FetcherMeta{Res: file, Opts: &base.Options{}}); got != nil { + t.Fatalf("no options, got %+v", got) + } +} diff --git a/pkg/download/model.go b/pkg/download/model.go index bf479a0..3005125 100644 --- a/pkg/download/model.go +++ b/pkg/download/model.go @@ -10,6 +10,7 @@ import ( "github.com/GopeedLab/gopeed/internal/production" "github.com/GopeedLab/gopeed/internal/protocol/bt" "github.com/GopeedLab/gopeed/internal/protocol/ed2k" + "github.com/GopeedLab/gopeed/internal/protocol/ftp" "github.com/GopeedLab/gopeed/internal/protocol/hls" "github.com/GopeedLab/gopeed/internal/protocol/http" "github.com/GopeedLab/gopeed/pkg/base" @@ -213,6 +214,10 @@ func (cfg *DownloaderConfig) Init() *DownloaderConfig { } if len(cfg.FetchManagers) == 0 { cfg.FetchManagers = []fetcher.FetcherManager{ + // FTP matches by URL scheme only. It comes before the managers that + // match by file extension, so ftp://host/x.torrent or x.m3u8 is + // downloaded as a file over FTP. + new(ftp.FetcherManager), new(hls.FetcherManager), new(http.FetcherManager), new(bt.FetcherManager), diff --git a/pkg/protocol/ftp/model.go b/pkg/protocol/ftp/model.go new file mode 100644 index 0000000..a35acfb --- /dev/null +++ b/pkg/protocol/ftp/model.go @@ -0,0 +1,49 @@ +package ftp + +// TLS modes for the "tls" option of an ftp:// task. ftps:// URLs always use +// implicit TLS and ftpes:// URLs always use explicit TLS. +const ( + TLSNone = "none" + TLSExplicit = "explicit" + TLSImplicit = "implicit" +) + +type OptsExtra struct { + // Connections is the maximum number of parallel logins. Servers often cap + // logins per user, so the fetcher lowers it when the server refuses one. + Connections int `json:"connections"` + // TLS upgrades a plain ftp:// URL: "none" (default), "explicit" (AUTH TLS) + // or "implicit". It is ignored for ftps:// and ftpes:// URLs. + TLS string `json:"tls"` + // AutoTorrent when task download complete, and it is a .torrent file, it will be auto create a new task for the torrent file + // nil means use global config, true/false means explicit setting + AutoTorrent *bool `json:"autoTorrent"` + // DeleteTorrentAfterDownload when true, deletes the .torrent file after creating BT task + // nil means use global config, true/false means explicit setting + DeleteTorrentAfterDownload *bool `json:"deleteTorrentAfterDownload"` + // AutoExtract when task download complete, and it is an archive file, it will be auto extracted + // nil means use global config, true/false means explicit setting + AutoExtract *bool `json:"autoExtract"` + // ArchivePassword is the password for extracting password-protected archives + ArchivePassword string `json:"archivePassword"` + // DeleteAfterExtract when true, deletes the archive file after successful extraction + DeleteAfterExtract bool `json:"deleteAfterExtract"` +} + +// Stats for download. It has the same shape as the HTTP protocol's stats, so +// clients can show FTP connections the way they show HTTP connections. +type Stats struct { + Connections []*StatsConnection `json:"connections"` +} + +// StatsConnection describes one login that took part in the download. +type StatsConnection struct { + Downloaded int64 `json:"downloaded"` + // Total is the resource size divided by the number of connections, + // rounded up to a whole byte, as in the HTTP protocol's stats. It is zero + // when the resource size is unknown. + Total int64 `json:"total"` + Completed bool `json:"completed"` + Failed bool `json:"failed"` + RetryTimes int `json:"retryTimes"` +} -- 2.55.0 From 13ae1402a39e03fe8db44a2870336ceb5a36b644 Mon Sep 17 00:00:00 2001 From: "Silas, Mariusz Grzybacz" Date: Fri, 2 Oct 2026 21:45:00 +0000 Subject: [PATCH 2/3] fix(ftp): keep refused segments owned, per-connection TLS sessions - A worker that finds the queue empty now leaves the run under the same lock the login-refusal path uses to count who is left. A segment put back by a late refusal could otherwise be stranded and fail the task with "data missing". - A worker refused after it had logged in (another client took the slot) now hands its segment back and lowers the ceiling like any other refusal, instead of retrying until the task fails. Only the last worker waits and retries. - Each control connection gets its own TLS session cache, so its data connections resume its own session. ProFTPD's mod_tls rejects a data connection that resumes another connection's session. - The base TLS config of the manager is unexported; only tests set it. - Tests: a delayed-refusal folder download, a login cap lowered mid-download, a server that enforces per-connection TLS session reuse, and a resume test that stops the server at a fixed byte count instead of relying on timing. --- internal/protocol/ftp/conn.go | 21 ++- internal/protocol/ftp/fetcher.go | 94 ++++++----- internal/protocol/ftp/fetcher_manager.go | 8 +- internal/protocol/ftp/fetcher_test.go | 138 ++++++++++++--- internal/protocol/ftp/testserver_test.go | 206 ++++++++++++++++++++++- 5 files changed, 392 insertions(+), 75 deletions(-) diff --git a/internal/protocol/ftp/conn.go b/internal/protocol/ftp/conn.go index 8b56559..2ebdce9 100644 --- a/internal/protocol/ftp/conn.go +++ b/internal/protocol/ftp/conn.go @@ -280,22 +280,27 @@ func (s *session) dialData(ctx context.Context, netDial dialContextFunc, tlsConf return raw, nil } -// tlsConfig returns the TLS settings for one server. Certificates are -// verified unless the request sets skipVerifyCert, as for HTTP. Data -// connections resume the control connection's TLS session through the shared -// cache, which servers such as vsftpd require by default. +// tlsConfig returns the TLS settings for one control connection and its data +// connections. Certificates are verified unless the request sets +// skipVerifyCert, as for HTTP. +// +// Each call gets its own one-entry session cache, so a data connection +// resumes the TLS session of its own control connection. vsftpd +// (require_ssl_reuse) only checks that some session was resumed, but ProFTPD's +// mod_tls also checks that it is the control connection's; a cache shared by +// several workers would hand a data connection another worker's session. +// A cache in the manager's base config is kept as given. func (f *Fetcher) tlsConfig(host string) *tls.Config { cfg := &tls.Config{} - if f.manager != nil && f.manager.TLSConfig != nil { - cfg = f.manager.TLSConfig.Clone() + if f.manager != nil && f.manager.baseTLS != nil { + cfg = f.manager.baseTLS.Clone() } cfg.ServerName = host if f.meta.Req != nil && f.meta.Req.SkipVerifyCert { cfg.InsecureSkipVerify = true } if cfg.ClientSessionCache == nil { - f.sessionCacheOnce.Do(func() { f.sessionCache = tls.NewLRUClientSessionCache(0) }) - cfg.ClientSessionCache = f.sessionCache + cfg.ClientSessionCache = tls.NewLRUClientSessionCache(1) } return cfg } diff --git a/internal/protocol/ftp/fetcher.go b/internal/protocol/ftp/fetcher.go index fb4bf6b..539d077 100644 --- a/internal/protocol/ftp/fetcher.go +++ b/internal/protocol/ftp/fetcher.go @@ -10,7 +10,6 @@ package ftp import ( "context" - "crypto/tls" "errors" "fmt" "io" @@ -87,9 +86,7 @@ type Fetcher struct { meta *fetcher.FetcherMeta doneCh chan error - sessionCacheOnce sync.Once - sessionCache tls.ClientSessionCache - warnOnce sync.Once + warnOnce sync.Once // loginOK is set once these credentials have logged in, after which a // 530 on another login means the server is full. loginOK atomic.Bool @@ -420,18 +417,22 @@ func (f *Fetcher) download(ctx context.Context, t *target, files map[int]*os.Fil } } -// pool hands out the segments of one run to its workers. +// pool hands out the segments of one run to its workers. Every decision to +// leave the run is taken under mu together with the queue it depends on, so +// a segment put back can never be left without a worker. type pool struct { - mu sync.Mutex - queue []*segment - running int // workers still in the run - loggedIn int // workers that logged in during the run + mu sync.Mutex + queue []*segment + running int // workers still in the run } +// take returns the next segment. A worker that finds the queue empty leaves +// the run. func (p *pool) take() *segment { p.mu.Lock() defer p.mu.Unlock() if len(p.queue) == 0 { + p.running-- return nil } s := p.queue[0] @@ -445,6 +446,28 @@ func (p *pool) putBack(s *segment) { p.queue = append([]*segment{s}, p.queue...) } +// refused puts back the segment of a worker whose login the server turned +// away. The worker leaves when another worker is still in the run to take +// the segment, and left is the number of workers that remain; the last +// worker stays and retries. +func (p *pool) refused(s *segment) (leave bool, left int) { + p.mu.Lock() + defer p.mu.Unlock() + p.queue = append([]*segment{s}, p.queue...) + if p.running > 1 { + p.running-- + return true, p.running + } + return false, p.running +} + +// leave takes a worker out of the run when it stops for any other reason. +func (p *pool) leave() { + p.mu.Lock() + defer p.mu.Unlock() + p.running-- +} + func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.File) error { runCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -500,9 +523,11 @@ func (f *Fetcher) runWorkers(ctx context.Context, t *target, files map[int]*os.F return nil } -// worker fetches segments from the pool until it is empty. A worker that the -// server turns away before it ever logged in lowers the ceiling and leaves, -// handing its segment to the workers that did log in. +// worker fetches segments from the pool until it is empty. A worker whose +// login the server refuses (421, or 530 once these credentials have worked) +// hands its segment back and leaves while other workers remain, lowering the +// task's ceiling to the workers left. This holds whether or not the worker +// had logged in before: a slot can also go to another client mid-download. func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, p *pool, id int) error { var ln *lane f.mu.Lock() @@ -510,15 +535,7 @@ func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, ln = f.lanes[id] } f.mu.Unlock() - loggedIn := false onLogin := func() { - if loggedIn { - return - } - loggedIn = true - p.mu.Lock() - p.loggedIn++ - p.mu.Unlock() f.mu.Lock() if ln == nil { ln = &lane{} @@ -539,31 +556,30 @@ func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, failures = 0 continue } - p.putBack(seg) if ctx.Err() != nil { + p.putBack(seg) + p.leave() return nil } - if errors.Is(err, errRestUnsupported) { - return err - } var le *loginError - refused := errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) - if refused && !loggedIn { - p.mu.Lock() - ceiling := max(1, p.loggedIn) - leave := p.running > ceiling - if leave { - p.running-- - } - p.mu.Unlock() - if leave { - f.lowerCeiling(ceiling) + if errors.As(err, &le) && isLoginRefusal(le.err, f.loginOK.Load()) { + if leave, left := p.refused(seg); leave { + f.lowerCeiling(left) return nil } - } else if !refused && (isPermanent(err) || errors.Is(err, errDiskWrite)) { - f.markFailed(ln) - return err + // The last worker: wait for a slot. + } else { + p.putBack(seg) + if errors.Is(err, errRestUnsupported) { + p.leave() + return err + } + if isPermanent(err) || errors.Is(err, errDiskWrite) { + f.markFailed(ln) + p.leave() + return err + } } failures++ @@ -577,10 +593,12 @@ func (f *Fetcher) worker(ctx context.Context, t *target, files map[int]*os.File, f.mu.Unlock() if failures > maxRetries { f.markFailed(ln) + p.leave() return fmt.Errorf("ftp: giving up after %d retries: %w", maxRetries, err) } select { case <-ctx.Done(): + p.leave() return nil case <-time.After(retryDelay * time.Duration(failures)): } diff --git a/internal/protocol/ftp/fetcher_manager.go b/internal/protocol/ftp/fetcher_manager.go index ef51be2..62e6e70 100644 --- a/internal/protocol/ftp/fetcher_manager.go +++ b/internal/protocol/ftp/fetcher_manager.go @@ -26,10 +26,10 @@ type fetcherData struct { // FetcherManager handles ftp://, ftps:// (implicit TLS) and ftpes:// (explicit // TLS) URLs. type FetcherManager struct { - // TLSConfig is the base for every TLS connection, for example to trust a - // private root. Nil means the system roots. The task's skipVerifyCert - // still turns verification off. - TLSConfig *tls.Config + // baseTLS is the base for every TLS connection. Nil means the system + // roots; the package's tests set it to trust their own certificate. The + // task's skipVerifyCert still turns verification off. + baseTLS *tls.Config } func (fm *FetcherManager) Name() string { diff --git a/internal/protocol/ftp/fetcher_test.go b/internal/protocol/ftp/fetcher_test.go index ba93ea7..81a96b3 100644 --- a/internal/protocol/ftp/fetcher_test.go +++ b/internal/protocol/ftp/fetcher_test.go @@ -32,7 +32,7 @@ const mib = 1 << 20 func testManager(srv *FTPServer) *FetcherManager { fm := &FetcherManager{} if srv != nil { - fm.TLSConfig = &tls.Config{RootCAs: srv.CA} + fm.baseTLS = &tls.Config{RootCAs: srv.CA} } return fm } @@ -423,28 +423,15 @@ func TestNoRestSequential(t *testing.T) { } } -// waitProgress waits until the fetcher has between lo and hi bytes. -func waitProgress(t *testing.T, f *Fetcher, lo, hi int64) { - t.Helper() - deadline := time.Now().Add(30 * time.Second) - for time.Now().Before(deadline) { - got := f.Progress().TotalDownloaded() - if got >= lo { - if got > hi { - t.Fatalf("progress %d passed %d before the pause; slow the server down", got, hi) - } - return - } - time.Sleep(5 * time.Millisecond) - } - t.Fatalf("progress stayed under %d", lo) -} - +// TestResumeAfterKill stops the server's transfers at a fixed byte count, +// waits until the client has written exactly those bytes, then pauses, saves, +// restores and finishes. No timing decides where the pause lands. func TestResumeAfterKill(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) - srv.ReadDelay.Store(int64(3 * time.Millisecond)) size := int64(4*mib + 777) want := srv.WriteFile(t, "resume.bin", RandomBytes(int(size), 10)) + stallAt := size / 2 + srv.StallAfter(stallAt) fm := testManager(srv) f := buildFetcher(fm) @@ -452,10 +439,22 @@ func TestResumeAfterKill(t *testing.T) { if err := f.Start(); err != nil { t.Fatal(err) } - waitProgress(t, f, size/5, size*4/5) + select { + case <-srv.Stalled(): + case <-time.After(30 * time.Second): + t.Fatal("the server never reached the stall point") + } + deadline := time.Now().Add(30 * time.Second) + for f.Progress().TotalDownloaded() != stallAt { + if time.Now().After(deadline) { + t.Fatalf("progress %d, want the %d bytes the server sent", f.Progress().TotalDownloaded(), stallAt) + } + time.Sleep(5 * time.Millisecond) + } if err := f.Pause(); err != nil { t.Fatal(err) } + srv.Release() blob := storeData(t, fm, f) var saved fetcherData @@ -470,12 +469,11 @@ func TestResumeAfterKill(t *testing.T) { resumeAt[seg.Begin+seg.Downloaded] = true } } - if savedTotal == 0 || savedTotal >= size { - t.Fatalf("saved progress %d of %d", savedTotal, size) + if savedTotal != stallAt { + t.Fatalf("saved progress %d, want %d", savedTotal, stallAt) } before := len(srv.RetrOffsets()) - srv.ReadDelay.Store(0) restored := restoreFetcher(t, fm, f.Meta(), blob) if got := restored.Progress().TotalDownloaded(); got != savedTotal { t.Fatalf("restored progress %d, want %d", got, savedTotal) @@ -494,6 +492,100 @@ func TestResumeAfterKill(t *testing.T) { } } +// TestLateRefusalAfterQueueDrained: worker B holds the second file's segment +// while the server takes its time to refuse B's login. Worker A finishes the +// first file meanwhile and finds the queue empty. B must not leave too, or +// nobody fetches the second file. +func TestLateRefusalAfterQueueDrained(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{MaxLogins: 1, RefuseDelay: 500 * time.Millisecond}) + want := map[string]string{ + "a.bin": srv.WriteFile(t, "dir/a.bin", RandomBytes(1000, 16)), + "b.bin": srv.WriteFile(t, "dir/b.bin", RandomBytes(1000, 17)), + } + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2}) + startAndWait(t, f) + for name, sum := range want { + checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum) + } + if srv.Refused() == 0 { + t.Fatal("the server never refused a login, so the late refusal was not exercised") + } +} + +// TestLoginCapLoweredMidway: both workers have logged in when the server's +// cap drops to one, as when another client takes a slot. The refused worker +// leaves, the ceiling drops to one, and the task finishes. +func TestLoginCapLoweredMidway(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{}) + srv.ReadDelay.Store(int64(time.Millisecond)) + want := map[string]string{} + for i := 0; i < 8; i++ { + name := fmt.Sprintf("f%d.bin", i) + want[name] = srv.WriteFile(t, "dir/"+name, RandomBytes(512<<10, byte(30+i))) + } + + fm := testManager(srv) + f := buildFetcher(fm) + resolveTask(t, f, ftpURL(srv, "ftp", "", "dir"), &pftp.OptsExtra{Connections: 2}) + if err := f.Start(); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(30 * time.Second) + for len(ftpStats(t, f).Connections) < 2 || f.Progress().TotalDownloaded() < 512<<10 { + if time.Now().After(deadline) { + t.Fatal("the two workers never got going") + } + time.Sleep(2 * time.Millisecond) + } + srv.SetMaxLogins(1) + if err := waitFetcher(t, f); err != nil { + t.Fatalf("download: %v", err) + } + for name, sum := range want { + checkFile(t, filepath.Join(f.Meta().FolderPath(), name), sum) + } + if srv.Refused() == 0 { + t.Fatal("the server never refused a login, so the lowered cap was not exercised") + } + for _, c := range ftpStats(t, f).Connections { + if c.Failed { + t.Fatalf("a refused worker failed: %+v", c) + } + } + var saved fetcherData + if err := json.Unmarshal(storeData(t, fm, f), &saved); err != nil { + t.Fatal(err) + } + if saved.MaxConns != 1 { + t.Fatalf("saved login ceiling %d, want 1", saved.MaxConns) + } +} + +// TestDataConnectionsResumeOwnSession runs four FTPS workers against a server +// that, like ProFTPD, accepts a data connection only when it resumes the TLS +// session of its own control connection. +func TestDataConnectionsResumeOwnSession(t *testing.T) { + srv := StartFTPServer(t, ServerOptions{TLS: pftp.TLSImplicit, RequireSessionReuse: true}) + srv.ReadDelay.Store(int64(200 * time.Microsecond)) + want := srv.WriteFile(t, "reuse.bin", RandomBytes(8*mib, 18)) + + f := buildFetcher(testManager(srv)) + resolveTask(t, f, ftpURL(srv, "ftps", "", "reuse.bin"), &pftp.OptsExtra{Connections: 4}) + startAndWait(t, f) + checkFile(t, f.Meta().SingleFilepath(), want) + ok, bad := srv.SessionReuse() + if ok < 4 || bad != 0 { + t.Fatalf("data connections: %d resumed their own session, %d did not", ok, bad) + } + for _, c := range ftpStats(t, f).Connections { + if c.RetryTimes != 0 { + t.Fatalf("a worker needed retries: %+v", c) + } + } +} + func TestRetryAfterDroppedConnection(t *testing.T) { srv := StartFTPServer(t, ServerOptions{}) want := srv.WriteFile(t, "drop.bin", RandomBytes(3*mib, 11)) diff --git a/internal/protocol/ftp/testserver_test.go b/internal/protocol/ftp/testserver_test.go index d26225a..2197bc0 100644 --- a/internal/protocol/ftp/testserver_test.go +++ b/internal/protocol/ftp/testserver_test.go @@ -16,6 +16,8 @@ import ( "net" "os" "path/filepath" + "regexp" + "strconv" "strings" "sync" "sync/atomic" @@ -45,6 +47,15 @@ type ServerOptions struct { // any login, anonymous included. User string Pass string + // RefuseDelay holds every 421 refusal back this long, like a server + // that answers slowly when it is full. + RefuseDelay time.Duration + // RequireSessionReuse makes every data connection resume the TLS + // session of the control connection that opened it, as ProFTPD's + // mod_tls does by default. A data connection that does not is refused. + // It needs TLSImplicit, where this test's wrapper sees the plain + // control stream and so learns which passive port belongs to whom. + RequireSessionReuse bool } // FTPServer is an ftpserverlib server over a temp directory. @@ -71,6 +82,20 @@ type FTPServer struct { retrs []int64 // offset of every RETR, 0 when no REST came first refused int logins int + + // Passive port to the control connection that announced it, and the + // outcome of every data connection's session check. + pasvOwner map[int]string + reuseOK int + reuseBad int + + // Stall state, see StallAfter. stallMu is held across file reads. + stallMu sync.Mutex + stallAt int64 + sent int64 + stallCh chan struct{} + stalled chan struct{} + stalledOnce *sync.Once } // StartFTPServer starts a server and stops it when the test ends. @@ -84,6 +109,12 @@ func StartFTPServer(t *testing.T, opts ServerOptions) *FTPServer { t.Fatal(err) } var ln net.Listener = base + if opts.RequireSessionReuse { + if opts.TLS != pftp.TLSImplicit { + t.Fatal("RequireSessionReuse needs TLSImplicit") + } + s.requireSessionReuse(t, base.Addr().(*net.TCPAddr).Port) + } if opts.TLS == pftp.TLSImplicit { ln = tls.NewListener(ln, s.tlsConf) } @@ -144,6 +175,163 @@ func (s *FTPServer) Logins() int { return s.logins } +// SetMaxLogins changes the login cap while the server runs, as when other +// clients take slots. +func (s *FTPServer) SetMaxLogins(n int) { + s.mu.Lock() + s.opts.MaxLogins = n + s.mu.Unlock() +} + +// SessionReuse returns how many data connections resumed their own control +// connection's TLS session, and how many did not. +func (s *FTPServer) SessionReuse() (ok, bad int) { + s.mu.Lock() + defer s.mu.Unlock() + return s.reuseOK, s.reuseBad +} + +// StallAfter makes every transfer stop sending once the server has sent n +// bytes in total, until Release. Stalled is closed when that happens. +func (s *FTPServer) StallAfter(n int64) { + s.stallMu.Lock() + defer s.stallMu.Unlock() + s.stallAt, s.sent = n, 0 + s.stallCh = make(chan struct{}) + s.stalled = make(chan struct{}) + s.stalledOnce = &sync.Once{} +} + +// Stalled is closed once a transfer has stopped at the StallAfter limit. +func (s *FTPServer) Stalled() <-chan struct{} { + s.stallMu.Lock() + defer s.stallMu.Unlock() + return s.stalled +} + +// Sent returns the bytes sent since StallAfter. +func (s *FTPServer) Sent() int64 { + s.stallMu.Lock() + defer s.stallMu.Unlock() + return s.sent +} + +// Release ends the stall. The stalled transfers fail. +func (s *FTPServer) Release() { + s.stallMu.Lock() + defer s.stallMu.Unlock() + if s.stallAt > 0 { + s.stallAt = 0 + close(s.stallCh) + } +} + +var errStalled = errors.New("test server released a stalled transfer") + +// stall limits a read to the StallAfter budget, and blocks once the budget +// is spent. +func (s *FTPServer) stall(read func([]byte) (int, error), p []byte) (int, error) { + s.stallMu.Lock() + if s.stallAt <= 0 { + s.stallMu.Unlock() + return read(p) + } + allowed := s.stallAt - s.sent + if allowed <= 0 { + ch := s.stallCh + s.stalledOnce.Do(func() { close(s.stalled) }) + s.stallMu.Unlock() + <-ch + return 0, errStalled + } + if int64(len(p)) > allowed { + p = p[:allowed] + } + n, err := read(p) + s.sent += int64(n) + s.stallMu.Unlock() + return n, err +} + +var pasvReply = regexp.MustCompile(`^(?:229 [^(]*\(\|\|\|(\d+)\|\)|227 [^(]*\(\d+,\d+,\d+,\d+,(\d+),(\d+)\))`) + +// notePassive records the passive port a control connection announced. +func (s *FTPServer) notePassive(owner string, reply []byte) { + m := pasvReply.FindSubmatch(reply) + if m == nil { + return + } + var port int + if len(m[1]) > 0 { + port, _ = strconv.Atoi(string(m[1])) + } else { + hi, _ := strconv.Atoi(string(m[2])) + lo, _ := strconv.Atoi(string(m[3])) + port = hi<<8 | lo + } + s.mu.Lock() + s.pasvOwner[port] = owner + s.mu.Unlock() +} + +// requireSessionReuse stamps each control connection's session tickets with +// its address, and checks that each data connection resumed a ticket of the +// control connection that announced its passive port. +func (s *FTPServer) requireSessionReuse(t *testing.T, controlPort int) { + base := s.tlsConf + var key [32]byte + if _, err := rand.Read(key[:]); err != nil { + t.Fatal(err) + } + base.SetSessionTicketKeys([][32]byte{key}) + s.pasvOwner = map[int]string{} + stamp := func(owner string) func(tls.ConnectionState, *tls.SessionState) ([]byte, error) { + return func(cs tls.ConnectionState, ss *tls.SessionState) ([]byte, error) { + ss.Extra = [][]byte{[]byte(owner)} + return base.EncryptTicket(cs, ss) + } + } + base.GetConfigForClient = func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + cfg := base.Clone() + cfg.GetConfigForClient = nil + local, _ := hello.Conn.LocalAddr().(*net.TCPAddr) + if local == nil || local.Port == controlPort { + cfg.WrapSession = stamp(hello.Conn.RemoteAddr().String()) + return cfg, nil + } + s.mu.Lock() + owner := s.pasvOwner[local.Port] + s.mu.Unlock() + var resumedFrom string + cfg.UnwrapSession = func(identity []byte, cs tls.ConnectionState) (*tls.SessionState, error) { + ss, err := base.DecryptTicket(identity, cs) + if err != nil || ss == nil { + return nil, nil + } + if len(ss.Extra) > 0 { + resumedFrom = string(ss.Extra[0]) + } + return ss, nil + } + cfg.WrapSession = stamp(owner) + cfg.VerifyConnection = func(cs tls.ConnectionState) error { + ok := owner != "" && cs.DidResume && resumedFrom == owner + s.mu.Lock() + if ok { + s.reuseOK++ + } else { + s.reuseBad++ + } + s.mu.Unlock() + if !ok { + return errors.New("data connection did not resume its control connection's TLS session") + } + return nil + } + return cfg, nil + } +} + func (s *FTPServer) acquire() bool { s.mu.Lock() defer s.mu.Unlock() @@ -176,13 +364,19 @@ func (l *limitListener) Accept() (net.Conn, error) { } if !l.srv.acquire() { go func() { + time.Sleep(l.srv.opts.RefuseDelay) _ = c.SetDeadline(time.Now().Add(5 * time.Second)) _, _ = io.WriteString(c, "421 Too many connections\r\n") _ = c.Close() }() continue } - var conn net.Conn = &countedConn{Conn: c, release: l.srv.release} + cc := &countedConn{Conn: c, release: l.srv.release} + if l.srv.opts.RequireSessionReuse { + owner := c.RemoteAddr().String() + cc.sniff = func(b []byte) { l.srv.notePassive(owner, b) } + } + var conn net.Conn = cc if l.srv.opts.NoRest { conn = &noRestConn{countedConn: conn.(*countedConn), in: bufio.NewReader(c)} } @@ -194,6 +388,14 @@ type countedConn struct { net.Conn once sync.Once release func() + sniff func([]byte) // sees every reply the server writes +} + +func (c *countedConn) Write(b []byte) (int, error) { + if c.sniff != nil { + c.sniff(b) + } + return c.Conn.Write(b) } func (c *countedConn) Close() error { @@ -316,7 +518,7 @@ func (f *slowFile) Read(p []byte) (int, error) { p = p[:rest] } } - n, err := f.File.Read(p) + n, err := f.srv.stall(f.File.Read, p) f.sent += int64(n) return n, err } -- 2.55.0 From c4fb84e19b162374023de7fc93f8f9a606f947db Mon Sep 17 00:00:00 2001 From: "Silas, Mariusz Grzybacz" Date: Fri, 2 Oct 2026 22:06:53 +0000 Subject: [PATCH 3/3] feat(ui): create FTP, FTPS and FTPES tasks The create task page recognises ftp://, ftps:// and ftpes:// links, checked before the .torrent suffix so ftp://host/file.torrent downloads the file over FTP. FTP tasks get their own options (OptsExtraFtp): connections capped at 32, and for plain ftp:// links a TLS select (none, explicit or implicit) that is sent as the "tls" option. ftps:// and ftpes:// fix the mode, so they send no "tls". The options are built per link, including the resolve, rid and batch paths, so a mixed list keeps HTTP links at up to 256 connections. Protocol gains ftp, so FTP tasks decode in the task list instead of failing the enum lookup, and FTP stats render with the HTTP connection layout they share. New strings ftpTls, ftpTlsNone, ftpTlsExplicit and ftpTlsImplicit are translated in every locale. --- ui/flutter/lib/api/model/options.dart | 28 +++ ui/flutter/lib/api/model/options.g.dart | 20 +++ ui/flutter/lib/api/model/task.dart | 2 +- ui/flutter/lib/api/model/task.g.dart | 8 +- ui/flutter/lib/api/model/task_stats.dart | 3 +- .../features/tasks/domain/task_record.dart | 2 +- .../pages/create_task_window_page.dart | 165 ++++++++++++++++-- ui/flutter/lib/l10n/app_ar.arb | 4 + ui/flutter/lib/l10n/app_az.arb | 4 + ui/flutter/lib/l10n/app_ca.arb | 4 + ui/flutter/lib/l10n/app_de.arb | 4 + ui/flutter/lib/l10n/app_en.arb | 4 + ui/flutter/lib/l10n/app_es.arb | 4 + ui/flutter/lib/l10n/app_fa.arb | 4 + ui/flutter/lib/l10n/app_fr.arb | 4 + ui/flutter/lib/l10n/app_hu.arb | 4 + ui/flutter/lib/l10n/app_id.arb | 4 + ui/flutter/lib/l10n/app_it.arb | 4 + ui/flutter/lib/l10n/app_ja.arb | 4 + ui/flutter/lib/l10n/app_ko.arb | 4 + ui/flutter/lib/l10n/app_pl.arb | 4 + ui/flutter/lib/l10n/app_pt.arb | 4 + ui/flutter/lib/l10n/app_ru.arb | 4 + ui/flutter/lib/l10n/app_sk.arb | 4 + ui/flutter/lib/l10n/app_ta.arb | 4 + ui/flutter/lib/l10n/app_tr.arb | 4 + ui/flutter/lib/l10n/app_uk.arb | 4 + ui/flutter/lib/l10n/app_vi.arb | 4 + ui/flutter/lib/l10n/app_zh.arb | 4 + ui/flutter/lib/l10n/app_zh_TW.arb | 4 + ui/flutter/test/api/task_stats_test.dart | 16 ++ .../tasks/domain/task_record_test.dart | 11 +- .../create_task_window_page_test.dart | 163 +++++++++++++++++ 33 files changed, 487 insertions(+), 23 deletions(-) diff --git a/ui/flutter/lib/api/model/options.dart b/ui/flutter/lib/api/model/options.dart index 17aadd0..0c73073 100644 --- a/ui/flutter/lib/api/model/options.dart +++ b/ui/flutter/lib/api/model/options.dart @@ -39,3 +39,31 @@ class OptsExtraHttp { Map toJson() => _$OptsExtraHttpToJson(this); } + +/// Options for an FTP task. It mirrors `pkg/protocol/ftp.OptsExtra`; [tls] +/// only applies to `ftp://` URLs, because `ftps://` always uses implicit TLS +/// and `ftpes://` always uses explicit TLS. +@JsonSerializable() +class OptsExtraFtp { + int connections; + String? tls; + bool? autoTorrent; + bool? deleteTorrentAfterDownload; + bool? autoExtract; + String archivePassword; + bool deleteAfterExtract; + + OptsExtraFtp({ + this.connections = 0, + this.tls, + this.autoTorrent, + this.deleteTorrentAfterDownload, + this.autoExtract, + this.archivePassword = '', + this.deleteAfterExtract = false, + }); + + factory OptsExtraFtp.fromJson(Map json) => _$OptsExtraFtpFromJson(json); + + Map toJson() => _$OptsExtraFtpToJson(this); +} diff --git a/ui/flutter/lib/api/model/options.g.dart b/ui/flutter/lib/api/model/options.g.dart index 52b8645..6e658bf 100644 --- a/ui/flutter/lib/api/model/options.g.dart +++ b/ui/flutter/lib/api/model/options.g.dart @@ -39,3 +39,23 @@ Map _$OptsExtraHttpToJson(OptsExtraHttp instance) => json) => OptsExtraFtp( + connections: (json['connections'] as num?)?.toInt() ?? 0, + tls: json['tls'] as String?, + autoTorrent: json['autoTorrent'] as bool?, + deleteTorrentAfterDownload: json['deleteTorrentAfterDownload'] as bool?, + autoExtract: json['autoExtract'] as bool?, + archivePassword: json['archivePassword'] as String? ?? '', + deleteAfterExtract: json['deleteAfterExtract'] as bool? ?? false, +); + +Map _$OptsExtraFtpToJson(OptsExtraFtp instance) => { + 'connections': instance.connections, + 'tls': ?instance.tls, + 'autoTorrent': ?instance.autoTorrent, + 'deleteTorrentAfterDownload': ?instance.deleteTorrentAfterDownload, + 'autoExtract': ?instance.autoExtract, + 'archivePassword': instance.archivePassword, + 'deleteAfterExtract': instance.deleteAfterExtract, +}; diff --git a/ui/flutter/lib/api/model/task.dart b/ui/flutter/lib/api/model/task.dart index e72c6aa..0ea877d 100644 --- a/ui/flutter/lib/api/model/task.dart +++ b/ui/flutter/lib/api/model/task.dart @@ -6,7 +6,7 @@ part 'task.g.dart'; enum Status { ready, running, pause, wait, error, done } -enum Protocol { http, bt, ed2k, hls } +enum Protocol { http, bt, ed2k, hls, ftp } // ExtractStatus enum matching Go backend enum ExtractStatus { diff --git a/ui/flutter/lib/api/model/task.g.dart b/ui/flutter/lib/api/model/task.g.dart index d394703..06974c3 100644 --- a/ui/flutter/lib/api/model/task.g.dart +++ b/ui/flutter/lib/api/model/task.g.dart @@ -40,7 +40,13 @@ const _$StatusEnumMap = { Status.done: 'done', }; -const _$ProtocolEnumMap = {Protocol.http: 'http', Protocol.bt: 'bt', Protocol.ed2k: 'ed2k', Protocol.hls: 'hls'}; +const _$ProtocolEnumMap = { + Protocol.http: 'http', + Protocol.bt: 'bt', + Protocol.ed2k: 'ed2k', + Protocol.hls: 'hls', + Protocol.ftp: 'ftp', +}; Progress _$ProgressFromJson(Map json) => Progress( used: (json['used'] as num).toInt(), diff --git a/ui/flutter/lib/api/model/task_stats.dart b/ui/flutter/lib/api/model/task_stats.dart index dfe6c8c..c3e1536 100644 --- a/ui/flutter/lib/api/model/task_stats.dart +++ b/ui/flutter/lib/api/model/task_stats.dart @@ -52,7 +52,8 @@ sealed class TaskStats { final runtime = _map(json['runtime']); if (snapshot.isEmpty && runtime.isEmpty) return null; return switch (protocol) { - Protocol.http => HttpTaskStats.fromJson(snapshot), + // FTP stats have the same shape as HTTP stats: one entry per connection. + Protocol.http || Protocol.ftp => HttpTaskStats.fromJson(snapshot), Protocol.bt => BtTaskStats.fromJson(snapshot: snapshot, runtime: runtime), Protocol.ed2k => Ed2kTaskStats.fromJson(snapshot: snapshot, runtime: runtime), Protocol.hls || null => null, diff --git a/ui/flutter/lib/features/tasks/domain/task_record.dart b/ui/flutter/lib/features/tasks/domain/task_record.dart index 85a10cb..516993a 100644 --- a/ui/flutter/lib/features/tasks/domain/task_record.dart +++ b/ui/flutter/lib/features/tasks/domain/task_record.dart @@ -304,7 +304,7 @@ TaskAssetType _assetType(String name, {required bool isFolder, required api_task return switch (protocol) { api_task.Protocol.bt => TaskAssetType.torrent, api_task.Protocol.ed2k => TaskAssetType.ed2k, - api_task.Protocol.hls || api_task.Protocol.http || null => TaskAssetType.file, + api_task.Protocol.hls || api_task.Protocol.http || api_task.Protocol.ftp || null => TaskAssetType.file, }; } diff --git a/ui/flutter/lib/features/tasks/presentation/pages/create_task_window_page.dart b/ui/flutter/lib/features/tasks/presentation/pages/create_task_window_page.dart index a1a4b5d..921cfe8 100644 --- a/ui/flutter/lib/features/tasks/presentation/pages/create_task_window_page.dart +++ b/ui/flutter/lib/features/tasks/presentation/pages/create_task_window_page.dart @@ -79,6 +79,9 @@ class _CreateTaskWindowPageState extends ConsumerState { bool? _deleteTorrentAfterDownload; bool? _autoExtract; bool _deleteAfterExtract = false; + String _ftpTls = _ftpTlsNone; + bool _allLinksFtp = false; + bool _hasPlainFtpLink = false; List _downloadCategories = const []; bool _showAdvanced = false; @@ -186,6 +189,25 @@ class _CreateTaskWindowPageState extends ConsumerState { } } _recognizeMagnetUri(urlText.trim()); + _refreshFtpLinks(); + } + + /// Shows the FTP options (the 32-connection limit and the TLS select) only + /// while the input contains FTP links. + void _refreshFtpLinks() { + final links = Util.textToLines(_urlController.text).map((line) => line.trim()).where((line) => line.isNotEmpty); + final allFtp = links.isNotEmpty && links.every((link) => _parseProtocol(link) == _TaskProtocol.ftp); + final hasPlainFtp = links.any(_isPlainFtpUrl); + if (allFtp == _allLinksFtp && hasPlainFtp == _hasPlainFtpLink) return; + if (!mounted) { + _allLinksFtp = allFtp; + _hasPlainFtpLink = hasPlainFtp; + return; + } + setState(() { + _allLinksFtp = allFtp; + _hasPlainFtpLink = hasPlainFtp; + }); } void _applyInitialTask(CreateTask? task) { @@ -200,6 +222,7 @@ class _CreateTaskWindowPageState extends ConsumerState { _httpBody = ''; _replaceHttpHeaders(const {'User-Agent': '', 'Cookie': '', 'Referer': ''}); _trackersController.clear(); + _ftpTls = _ftpTlsNone; _protocolTab = 0; _urlController.text = req!.url; _initialRawUrl = req.rawUrl; @@ -224,6 +247,7 @@ class _CreateTaskWindowPageState extends ConsumerState { _trackersController.text = extra.trackers.join('\n'); } break; + case _TaskProtocol.ftp: case _TaskProtocol.ed2k: case null: break; @@ -243,6 +267,10 @@ class _CreateTaskWindowPageState extends ConsumerState { _autoExtract = extra.autoExtract; _archivePasswordController.text = extra.archivePassword; _deleteAfterExtract = extra.deleteAfterExtract; + final tls = api_options.OptsExtraFtp.fromJson(Map.from(opts.extra! as Map)).tls; + if (_ftpTlsModes.contains(tls)) { + _ftpTls = tls!; + } } } }); @@ -388,10 +416,22 @@ class _CreateTaskWindowPageState extends ConsumerState { fieldKey: const ValueKey('create-task-connections-input'), controller: _connectionsController, min: 1, - max: 256, + max: _allLinksFtp ? _ftpMaxConnections : 256, hintText: context.l10n.enterCount, ), ), + if (_hasPlainFtpLink) ...[ + const SizedBox(height: 16), + AppFormRow( + direction: stacked ? Axis.vertical : Axis.horizontal, + label: context.l10n.ftpTls, + child: _FtpTlsSelect( + key: const ValueKey('create-task-ftp-tls'), + value: _ftpTls, + onChanged: (value) => setState(() => _ftpTls = value), + ), + ), + ], const SizedBox(height: 16), AppFormRow( direction: stacked ? Axis.vertical : Axis.horizontal, @@ -874,7 +914,10 @@ class _CreateTaskWindowPageState extends ConsumerState { .createTask( CreateTask( req: _buildRequest(url, protocolUrl: _protocolUrlFor(url)), - opts: _buildOptions(name: urls.length > 1 ? '' : _renameController.text.trim()), + opts: _buildOptions( + url: _protocolUrlFor(url), + name: urls.length > 1 ? '' : _renameController.text.trim(), + ), ), ); }), @@ -884,7 +927,7 @@ class _CreateTaskWindowPageState extends ConsumerState { } final request = _buildRequest(urls.first, protocolUrl: _protocolUrlFor(urls.first)); - final options = _buildOptions(); + final options = _buildOptions(url: _protocolUrlFor(urls.first)); final result = await ref.read(gopeedServiceProvider).resolve(ResolveTask(req: request, opts: options)); if (!mounted) return; final created = await _showResolveDialog(request, result); @@ -935,6 +978,7 @@ class _CreateTaskWindowPageState extends ConsumerState { extra = ReqExtraBt(trackers: trackers).toJson(); } break; + case _TaskProtocol.ftp: case _TaskProtocol.ed2k: case null: break; @@ -991,6 +1035,10 @@ class _CreateTaskWindowPageState extends ConsumerState { if (uppercaseUrl.startsWith('HTTP:') || uppercaseUrl.startsWith('HTTPS:')) { return _TaskProtocol.http; } + // Checked before BitTorrent, so ftp://host/file.torrent downloads the file over FTP. + if (uppercaseUrl.startsWith('FTP:') || uppercaseUrl.startsWith('FTPS:') || uppercaseUrl.startsWith('FTPES:')) { + return _TaskProtocol.ftp; + } if (uppercaseUrl.startsWith('MAGNET:') || uppercaseUrl.endsWith('.TORRENT')) { return _TaskProtocol.bt; } @@ -1000,6 +1048,9 @@ class _CreateTaskWindowPageState extends ConsumerState { return null; } + /// Whether [url] is a plain `ftp://` link, the only FTP scheme whose TLS mode is chosen by the user. + bool _isPlainFtpUrl(String url) => url.trim().toUpperCase().startsWith('FTP:'); + String _normalizeTaskUrl(String url) { if (_isBtHash(url)) { return 'magnet:?xt=urn:btih:$url'; @@ -1022,21 +1073,37 @@ class _CreateTaskWindowPageState extends ConsumerState { return text.length == 40 && RegExp(r'^[0-9a-fA-F]+$').hasMatch(text); } - api_options.Options _buildOptions({String? name, List selectFiles = const []}) { + /// Builds the task options for [url]. FTP links get the FTP options, with at + /// most [_ftpMaxConnections] connections and, for `ftp://`, the TLS mode. + api_options.Options _buildOptions({required String url, String? name, List selectFiles = const []}) { final connections = int.tryParse(_connectionsController.text.trim()) ?? 0; - return api_options.Options( - name: name ?? _renameController.text.trim(), - path: _directoryController.text.trim(), - asDefaultPath: _asDefaultPath, - selectFiles: selectFiles, - extra: api_options.OptsExtraHttp( + final Map extra; + if (_parseProtocol(url) == _TaskProtocol.ftp) { + extra = api_options.OptsExtraFtp( + connections: connections.clamp(0, _ftpMaxConnections), + tls: _isPlainFtpUrl(url) ? _ftpTls : null, + autoTorrent: _autoTorrent, + deleteTorrentAfterDownload: _deleteTorrentAfterDownload, + autoExtract: _autoExtract, + archivePassword: _archivePasswordController.text, + deleteAfterExtract: _deleteAfterExtract, + ).toJson(); + } else { + extra = api_options.OptsExtraHttp( connections: connections, autoTorrent: _autoTorrent, deleteTorrentAfterDownload: _deleteTorrentAfterDownload, autoExtract: _autoExtract, archivePassword: _archivePasswordController.text, deleteAfterExtract: _deleteAfterExtract, - ).toJson(), + ).toJson(); + } + return api_options.Options( + name: name ?? _renameController.text.trim(), + path: _directoryController.text.trim(), + asDefaultPath: _asDefaultPath, + selectFiles: selectFiles, + extra: extra, ); } @@ -1056,7 +1123,7 @@ class _CreateTaskWindowPageState extends ConsumerState { .createTask( CreateTask( rid: result.id, - opts: _buildOptions(selectFiles: selected), + opts: _buildOptions(url: request.url, selectFiles: selected), ), ); return; @@ -1065,20 +1132,35 @@ class _CreateTaskWindowPageState extends ConsumerState { if (result.res.files.isNotEmpty) { final reqs = selected.map((index) { final file = result.res.files[index]; + final fileRequest = file.req ?? request; return CreateTaskBatchItem( - req: file.req ?? request, + req: fileRequest, opts: api_options.Options( name: file.name, path: path.join(_directoryController.text.trim(), result.res.name, file.path), - extra: _buildOptions().extra, + extra: _buildOptions(url: fileRequest.url).extra, ), ); }).toList(); - await ref.read(gopeedServiceProvider).createTaskBatch(CreateTaskBatch(reqs: reqs, opts: _buildOptions())); + await ref + .read(gopeedServiceProvider) + .createTaskBatch( + CreateTaskBatch( + reqs: reqs, + opts: _buildOptions(url: request.url), + ), + ); return; } - await ref.read(gopeedServiceProvider).createTask(CreateTask(req: request, opts: _buildOptions())); + await ref + .read(gopeedServiceProvider) + .createTask( + CreateTask( + req: request, + opts: _buildOptions(url: request.url), + ), + ); } Future _showResolveDialog(Request request, ResolveResult result) async { @@ -1388,7 +1470,48 @@ class _CreateTaskWindowPageState extends ConsumerState { } } -enum _TaskProtocol { http, bt, ed2k } +enum _TaskProtocol { http, bt, ed2k, ftp } + +/// Chooses how a plain `ftp://` link is encrypted. The values are the +/// `tls` option of an FTP task. +class _FtpTlsSelect extends StatelessWidget { + const _FtpTlsSelect({super.key, required this.value, required this.onChanged}); + + final String value; + final ValueChanged onChanged; + + @override + Widget build(BuildContext context) { + final labels = { + _ftpTlsNone: context.l10n.ftpTlsNone, + _ftpTlsExplicit: context.l10n.ftpTlsExplicit, + _ftpTlsImplicit: context.l10n.ftpTlsImplicit, + }; + return LayoutBuilder( + builder: (context, constraints) => Select( + value: value, + constraints: BoxConstraints.tightFor(width: constraints.maxWidth), + itemBuilder: (context, selected) => + Text(labels[selected] ?? selected, maxLines: 1, overflow: TextOverflow.ellipsis), + popup: (context) => SelectPopup( + items: SelectItemList( + children: [ + for (final entry in labels.entries) + SelectItemButton( + key: ValueKey('create-task-ftp-tls-${entry.key}'), + value: entry.key, + child: Text(entry.value), + ), + ], + ), + ), + onChanged: (selected) { + if (selected != null) onChanged(selected); + }, + ), + ); + } +} class _DirectDownloadToggle extends StatelessWidget { const _DirectDownloadToggle({required this.value, required this.onChanged}); @@ -1766,3 +1889,11 @@ const _advancedScrollDuration = Duration(milliseconds: 260); const _advancedScrollExtentRetryDelay = Duration(milliseconds: 40); const _advancedScrollExtentChecks = 4; const _advancedScrollDistance = 112.0; + +/// The highest number of parallel logins offered for FTP links. FTP servers +/// usually allow only a few logins per user. +const _ftpMaxConnections = 32; +const _ftpTlsNone = 'none'; +const _ftpTlsExplicit = 'explicit'; +const _ftpTlsImplicit = 'implicit'; +const _ftpTlsModes = {_ftpTlsNone, _ftpTlsExplicit, _ftpTlsImplicit}; diff --git a/ui/flutter/lib/l10n/app_ar.arb b/ui/flutter/lib/l10n/app_ar.arb index f25ea37..cf16063 100644 --- a/ui/flutter/lib/l10n/app_ar.arb +++ b/ui/flutter/lib/l10n/app_ar.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "حفظ كمجلد التنزيل الافتراضي", "downloadDirValid": "الرجاء تحديد مجلد التنزيل", "connections": "الاتصالات", + "ftpTls": "تشفير FTP", + "ftpTlsNone": "بدون", + "ftpTlsExplicit": "TLS صريح (FTPES)", + "ftpTlsImplicit": "TLS ضمني (FTPS)", "maxRetries": "الحد الأقصى لإعادة المحاولات", "useServerCtime": "استخدام وقت الخادم لإنشاء الملف", "maxRunning": "الحد الأقصى للمهام المتزامنة", diff --git a/ui/flutter/lib/l10n/app_az.arb b/ui/flutter/lib/l10n/app_az.arb index 4a74c52..7077e8d 100644 --- a/ui/flutter/lib/l10n/app_az.arb +++ b/ui/flutter/lib/l10n/app_az.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Standart endirmə qovluğu kimi yadda saxla", "downloadDirValid": "Zəhmət olmasa endirmə qovluğunu seçin", "connections": "Əlaqələr", + "ftpTls": "FTP şifrələməsi", + "ftpTlsNone": "Yoxdur", + "ftpTlsExplicit": "Açıq TLS (FTPES)", + "ftpTlsImplicit": "Gizli TLS (FTPS)", "maxRetries": "Maksimum cəhd sayı", "useServerCtime": "Fayl yaratma tarixi üçün server vaxtından istifadə et", "maxRunning": "Paralel Endirmələr", diff --git a/ui/flutter/lib/l10n/app_ca.arb b/ui/flutter/lib/l10n/app_ca.arb index a1ddc64..5caed15 100644 --- a/ui/flutter/lib/l10n/app_ca.arb +++ b/ui/flutter/lib/l10n/app_ca.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Desa com a carpeta de baixades predeterminada", "downloadDirValid": "Seleccioneu la carpeta de baixades", "connections": "Connexions", + "ftpTls": "Xifratge FTP", + "ftpTlsNone": "Cap", + "ftpTlsExplicit": "TLS explícit (FTPES)", + "ftpTlsImplicit": "TLS implícit (FTPS)", "maxRetries": "Reintents màxims", "useServerCtime": "Utilitza l'hora del servidor per a la creació de fitxers", "maxRunning": "Màxim de tasques en execució", diff --git a/ui/flutter/lib/l10n/app_de.arb b/ui/flutter/lib/l10n/app_de.arb index 81f3fc7..e1322d0 100644 --- a/ui/flutter/lib/l10n/app_de.arb +++ b/ui/flutter/lib/l10n/app_de.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Als Standard-Downloadordner speichern", "downloadDirValid": "Bitte wählen Sie das Download-Verzeichnis", "connections": "Anschlüsse", + "ftpTls": "FTP-Verschlüsselung", + "ftpTlsNone": "Keine", + "ftpTlsExplicit": "Explizites TLS (FTPES)", + "ftpTlsImplicit": "Implizites TLS (FTPS)", "maxRetries": "Maximale Wiederholungsversuche", "useServerCtime": "Serverzeit für die Dateierstellung verwenden", "maxRunning": "Maximale Anzahl ausgeführter Aufgaben", diff --git a/ui/flutter/lib/l10n/app_en.arb b/ui/flutter/lib/l10n/app_en.arb index a91b498..11ce25e 100644 --- a/ui/flutter/lib/l10n/app_en.arb +++ b/ui/flutter/lib/l10n/app_en.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Save as default download directory", "downloadDirValid": "Please select the download directory", "connections": "Connections", + "ftpTls": "FTP encryption", + "ftpTlsNone": "None", + "ftpTlsExplicit": "Explicit TLS (FTPES)", + "ftpTlsImplicit": "Implicit TLS (FTPS)", "maxRetries": "Max retries", "useServerCtime": "Use server time for file creation", "maxRunning": "Concurrent Downloads", diff --git a/ui/flutter/lib/l10n/app_es.arb b/ui/flutter/lib/l10n/app_es.arb index 002c859..0b3b02b 100644 --- a/ui/flutter/lib/l10n/app_es.arb +++ b/ui/flutter/lib/l10n/app_es.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Guardar como carpeta de descargas predeterminada", "downloadDirValid": "Por favor, selecciona el directorio de descarga", "connections": "Conexiones", + "ftpTls": "Cifrado FTP", + "ftpTlsNone": "Ninguno", + "ftpTlsExplicit": "TLS explícito (FTPES)", + "ftpTlsImplicit": "TLS implícito (FTPS)", "maxRetries": "Reintentos máximos", "useServerCtime": "Usar tiempo del servidor para la creación de archivos", "maxRunning": "Máximo de Tareas en Ejecución", diff --git a/ui/flutter/lib/l10n/app_fa.arb b/ui/flutter/lib/l10n/app_fa.arb index 59a17a5..cd6be66 100644 --- a/ui/flutter/lib/l10n/app_fa.arb +++ b/ui/flutter/lib/l10n/app_fa.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "ذخیره به‌عنوان پوشه پیش‌فرض دانلود", "downloadDirValid": "لطفا دایرکتوری دانلود را انتخاب کنید", "connections": "اتصالات", + "ftpTls": "رمزگذاری FTP", + "ftpTlsNone": "هیچ", + "ftpTlsExplicit": "TLS صریح (FTPES)", + "ftpTlsImplicit": "TLS ضمنی (FTPS)", "maxRetries": "حداکثر تلاش مجدد", "useServerCtime": "استفاده از زمان سرور برای ایجاد فایل", "maxRunning": "حداکثر وظایف در حال اجرا", diff --git a/ui/flutter/lib/l10n/app_fr.arb b/ui/flutter/lib/l10n/app_fr.arb index 6627443..1624e75 100644 --- a/ui/flutter/lib/l10n/app_fr.arb +++ b/ui/flutter/lib/l10n/app_fr.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Enregistrer comme dossier de téléchargement par défaut", "downloadDirValid": "Veuillez sélectionner le répertoire de téléchargement", "connections": "Connexions", + "ftpTls": "Chiffrement FTP", + "ftpTlsNone": "Aucun", + "ftpTlsExplicit": "TLS explicite (FTPES)", + "ftpTlsImplicit": "TLS implicite (FTPS)", "maxRetries": "Nombre maximal de tentatives", "useServerCtime": "Utiliser l'heure du serveur pour la création de fichier", "maxRunning": "Tâches en cours maximum", diff --git a/ui/flutter/lib/l10n/app_hu.arb b/ui/flutter/lib/l10n/app_hu.arb index d0fe309..556f446 100644 --- a/ui/flutter/lib/l10n/app_hu.arb +++ b/ui/flutter/lib/l10n/app_hu.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Mentés alapértelmezett letöltési mappaként", "downloadDirValid": "Kérjük, válassza ki a letöltési könyvtárat", "connections": "Kapcsolatok", + "ftpTls": "FTP-titkosítás", + "ftpTlsNone": "Nincs", + "ftpTlsExplicit": "Explicit TLS (FTPES)", + "ftpTlsImplicit": "Implicit TLS (FTPS)", "maxRetries": "Maximális újrapróbálkozások", "useServerCtime": "Használja a szerveridőt a fájl létrehozásához", "maxRunning": "Maximális futó feladatok", diff --git a/ui/flutter/lib/l10n/app_id.arb b/ui/flutter/lib/l10n/app_id.arb index 9ad8edd..1cc329d 100644 --- a/ui/flutter/lib/l10n/app_id.arb +++ b/ui/flutter/lib/l10n/app_id.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Simpan sebagai folder unduhan default", "downloadDirValid": "Silahkan pilih direktori unduhan", "connections": "Koneksi", + "ftpTls": "Enkripsi FTP", + "ftpTlsNone": "Tidak ada", + "ftpTlsExplicit": "TLS eksplisit (FTPES)", + "ftpTlsImplicit": "TLS implisit (FTPS)", "maxRetries": "Percobaan ulang maks.", "useServerCtime": "Gunakan waktu server untuk pembuatan file", "maxRunning": "Tugas Maksimum yang Berjalan", diff --git a/ui/flutter/lib/l10n/app_it.arb b/ui/flutter/lib/l10n/app_it.arb index 0500ce2..4ffc365 100644 --- a/ui/flutter/lib/l10n/app_it.arb +++ b/ui/flutter/lib/l10n/app_it.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Salva come cartella di download predefinita", "downloadDirValid": "Seleziona la directory di scaricamento", "connections": "Connessioni", + "ftpTls": "Crittografia FTP", + "ftpTlsNone": "Nessuna", + "ftpTlsExplicit": "TLS esplicito (FTPES)", + "ftpTlsImplicit": "TLS implicito (FTPS)", "maxRetries": "Tentativi massimi", "useServerCtime": "Utilizza l'ora del server per la creazione dei file", "maxRunning": "Numero massimo di attività in esecuzione", diff --git a/ui/flutter/lib/l10n/app_ja.arb b/ui/flutter/lib/l10n/app_ja.arb index 2544161..078aaf5 100644 --- a/ui/flutter/lib/l10n/app_ja.arb +++ b/ui/flutter/lib/l10n/app_ja.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "デフォルトのダウンロード先として保存", "downloadDirValid": "ダウンロードディレクトリを選択してください", "connections": "接続", + "ftpTls": "FTP 暗号化", + "ftpTlsNone": "なし", + "ftpTlsExplicit": "明示的 TLS (FTPES)", + "ftpTlsImplicit": "暗黙的 TLS (FTPS)", "maxRetries": "最大リトライ回数", "useServerCtime": "ファイル作成時にサーバー時間を使用する", "maxRunning": "最大実行タスク", diff --git a/ui/flutter/lib/l10n/app_ko.arb b/ui/flutter/lib/l10n/app_ko.arb index 504b25b..1fcbbc4 100644 --- a/ui/flutter/lib/l10n/app_ko.arb +++ b/ui/flutter/lib/l10n/app_ko.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "기본 다운로드 폴더로 저장", "downloadDirValid": "다운로드 폴더를 선택해 주세요", "connections": "연결 수", + "ftpTls": "FTP 암호화", + "ftpTlsNone": "없음", + "ftpTlsExplicit": "명시적 TLS (FTPES)", + "ftpTlsImplicit": "암시적 TLS (FTPS)", "maxRetries": "최대 재시도 횟수", "useServerCtime": "파일 생성 시간에 서버 시간 사용", "maxRunning": "최대 동시 작업 수", diff --git a/ui/flutter/lib/l10n/app_pl.arb b/ui/flutter/lib/l10n/app_pl.arb index 4d512aa..04f76a8 100644 --- a/ui/flutter/lib/l10n/app_pl.arb +++ b/ui/flutter/lib/l10n/app_pl.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Zapisz jako domyślny folder pobierania", "downloadDirValid": "Wskaż folder pobierania", "connections": "Połączenia", + "ftpTls": "Szyfrowanie FTP", + "ftpTlsNone": "Brak", + "ftpTlsExplicit": "Jawne TLS (FTPES)", + "ftpTlsImplicit": "Niejawne TLS (FTPS)", "maxRetries": "Maks. liczba ponownych prób", "useServerCtime": "Użyj czasu serwera do utworzenia pliku", "maxRunning": "Maksymalna liczba zadań", diff --git a/ui/flutter/lib/l10n/app_pt.arb b/ui/flutter/lib/l10n/app_pt.arb index f93da09..09af0be 100644 --- a/ui/flutter/lib/l10n/app_pt.arb +++ b/ui/flutter/lib/l10n/app_pt.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Salvar como pasta de download padrão", "downloadDirValid": "Por favor, selecione a pasta", "connections": "Conexões", + "ftpTls": "Criptografia FTP", + "ftpTlsNone": "Nenhuma", + "ftpTlsExplicit": "TLS explícito (FTPES)", + "ftpTlsImplicit": "TLS implícito (FTPS)", "maxRetries": "Tentativas máximas", "useServerCtime": "Usar o tempo de servidor para a criação de ficheiros", "maxRunning": "Máximo de Tarefas Simultâneas", diff --git a/ui/flutter/lib/l10n/app_ru.arb b/ui/flutter/lib/l10n/app_ru.arb index 57035b2..a6dd6ae 100644 --- a/ui/flutter/lib/l10n/app_ru.arb +++ b/ui/flutter/lib/l10n/app_ru.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Сохранить как папку загрузок по умолчанию", "downloadDirValid": "Пожалуйста, выберите папку для загрузки", "connections": "Соединений", + "ftpTls": "Шифрование FTP", + "ftpTlsNone": "Нет", + "ftpTlsExplicit": "Явный TLS (FTPES)", + "ftpTlsImplicit": "Неявный TLS (FTPS)", "maxRetries": "Макс. число повторов", "useServerCtime": "Использовать время сервера для создания файла", "maxRunning": "Максимальное количество активных задач", diff --git a/ui/flutter/lib/l10n/app_sk.arb b/ui/flutter/lib/l10n/app_sk.arb index 1a7b5ca..228af49 100644 --- a/ui/flutter/lib/l10n/app_sk.arb +++ b/ui/flutter/lib/l10n/app_sk.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Uložiť ako predvolený priečinok na stiahnutie", "downloadDirValid": "Vyberte priečinok na stiahnutie", "connections": "Pripojenia", + "ftpTls": "Šifrovanie FTP", + "ftpTlsNone": "Žiadne", + "ftpTlsExplicit": "Explicitné TLS (FTPES)", + "ftpTlsImplicit": "Implicitné TLS (FTPS)", "maxRetries": "Maximálny počet pokusov", "useServerCtime": "Použiť čas servera na vytvorenie súboru", "maxRunning": "Súbežné sťahovania", diff --git a/ui/flutter/lib/l10n/app_ta.arb b/ui/flutter/lib/l10n/app_ta.arb index 04193bd..43b3a1a 100644 --- a/ui/flutter/lib/l10n/app_ta.arb +++ b/ui/flutter/lib/l10n/app_ta.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "இயல்புநிலை பதிவிறக்க கோப்புறையாக சேமி", "downloadDirValid": "பதிவிரக்குமிடத்தை தேர்வு செயுங்கள்", "connections": "தொடர்புகள்", + "ftpTls": "FTP குறியாக்கம்", + "ftpTlsNone": "இல்லை", + "ftpTlsExplicit": "வெளிப்படையான TLS (FTPES)", + "ftpTlsImplicit": "மறைமுக TLS (FTPS)", "maxRetries": "அதிகபட்ச மறுமுயற்சிகள்", "useServerCtime": "கோப்பு உருவாக்க சேவையக நேரத்தைப் பயன்படுத்தவும்", "maxRunning": "அதிகபட்ச இயங்கும் பணிகள்", diff --git a/ui/flutter/lib/l10n/app_tr.arb b/ui/flutter/lib/l10n/app_tr.arb index 7c01725..d630111 100644 --- a/ui/flutter/lib/l10n/app_tr.arb +++ b/ui/flutter/lib/l10n/app_tr.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Varsayılan indirme klasörü olarak kaydet", "downloadDirValid": "Lütfen indirme dizinini seçin", "connections": "Bağlantılar", + "ftpTls": "FTP şifrelemesi", + "ftpTlsNone": "Yok", + "ftpTlsExplicit": "Açık TLS (FTPES)", + "ftpTlsImplicit": "Örtük TLS (FTPS)", "maxRetries": "Maksimum yeniden deneme", "useServerCtime": "Dosya oluşturma tarihi için sunucu zamanını kullan", "maxRunning": "Maksimum Çalışan İşlem", diff --git a/ui/flutter/lib/l10n/app_uk.arb b/ui/flutter/lib/l10n/app_uk.arb index 65fc23a..44332f6 100644 --- a/ui/flutter/lib/l10n/app_uk.arb +++ b/ui/flutter/lib/l10n/app_uk.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Зберегти як типову папку завантажень", "downloadDirValid": "Будь ласка, виберіть каталог завантажень", "connections": "З’єднання", + "ftpTls": "Шифрування FTP", + "ftpTlsNone": "Немає", + "ftpTlsExplicit": "Явний TLS (FTPES)", + "ftpTlsImplicit": "Неявний TLS (FTPS)", "maxRetries": "Макс. кількість повторів", "useServerCtime": "Використовувати час сервера для створення файлу", "maxRunning": "Макс. завдань у черзі", diff --git a/ui/flutter/lib/l10n/app_vi.arb b/ui/flutter/lib/l10n/app_vi.arb index 8cbd839..6575bba 100644 --- a/ui/flutter/lib/l10n/app_vi.arb +++ b/ui/flutter/lib/l10n/app_vi.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "Lưu làm thư mục tải xuống mặc định", "downloadDirValid": "Vui lòng chọn thư mục tải về", "connections": "Kết nối", + "ftpTls": "Mã hóa FTP", + "ftpTlsNone": "Không", + "ftpTlsExplicit": "TLS tường minh (FTPES)", + "ftpTlsImplicit": "TLS ngầm định (FTPS)", "maxRetries": "Số lần thử lại tối đa", "useServerCtime": "Sử dụng thời gian máy chủ cho việc tạo tệp", "maxRunning": "Số nhiệm vụ tối đa", diff --git a/ui/flutter/lib/l10n/app_zh.arb b/ui/flutter/lib/l10n/app_zh.arb index 8f7505f..f5e8472 100644 --- a/ui/flutter/lib/l10n/app_zh.arb +++ b/ui/flutter/lib/l10n/app_zh.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "保存为默认下载目录", "downloadDirValid": "请选择下载目录", "connections": "连接数", + "ftpTls": "FTP 加密", + "ftpTlsNone": "无", + "ftpTlsExplicit": "显式 TLS (FTPES)", + "ftpTlsImplicit": "隐式 TLS (FTPS)", "maxRetries": "最大重试次数", "useServerCtime": "创建文件使用服务器时间", "maxRunning": "最大下载数", diff --git a/ui/flutter/lib/l10n/app_zh_TW.arb b/ui/flutter/lib/l10n/app_zh_TW.arb index 9d8a4b9..f518124 100644 --- a/ui/flutter/lib/l10n/app_zh_TW.arb +++ b/ui/flutter/lib/l10n/app_zh_TW.arb @@ -38,6 +38,10 @@ "rememberLastDownloadDir": "儲存為預設下載目錄", "downloadDirValid": "請選擇下載目錄", "connections": "連接數", + "ftpTls": "FTP 加密", + "ftpTlsNone": "無", + "ftpTlsExplicit": "明確 TLS (FTPES)", + "ftpTlsImplicit": "隱含 TLS (FTPS)", "maxRetries": "最大重試次數", "useServerCtime": "使用伺服器時間作為檔案建立時間", "maxRunning": "最大執行任務數", diff --git a/ui/flutter/test/api/task_stats_test.dart b/ui/flutter/test/api/task_stats_test.dart index 0b7b79e..bfad6b7 100644 --- a/ui/flutter/test/api/task_stats_test.dart +++ b/ui/flutter/test/api/task_stats_test.dart @@ -23,6 +23,22 @@ void main() { expect(stats.connections.last.completed, isTrue); }); + test('FTP stats use the HTTP connection layout', () { + final stats = + TaskStats.fromJson(Protocol.ftp, { + 'snapshot': { + 'connections': [ + {'downloaded': 512, 'total': 1024, 'completed': false, 'failed': false, 'retryTimes': 1}, + ], + }, + 'runtime': null, + }) + as HttpTaskStats; + + expect(stats.connections.single.downloaded, 512); + expect(stats.connections.single.retryTimes, 1); + }); + test('BT stats decode the ordered completion bitset and peer transport', () { final stats = TaskStats.fromJson(Protocol.bt, { diff --git a/ui/flutter/test/features/tasks/domain/task_record_test.dart b/ui/flutter/test/features/tasks/domain/task_record_test.dart index eba26e4..7bcdb0b 100644 --- a/ui/flutter/test/features/tasks/domain/task_record_test.dart +++ b/ui/flutter/test/features/tasks/domain/task_record_test.dart @@ -4,10 +4,10 @@ import 'package:gopeed/core/icons/gopeed_icons.dart'; import 'package:gopeed/features/tasks/domain/task_record.dart'; void main() { - api.Task buildTask(List selection, int size, {String? doneAt}) => api.Task.fromJson({ + api.Task buildTask(List selection, int size, {String? doneAt, String protocol = 'bt'}) => api.Task.fromJson({ 'id': 'bt-selection', 'name': 'bundle', - 'protocol': 'bt', + 'protocol': protocol, 'status': 'running', 'uploading': false, 'createdAt': '2026-01-01T00:00:00Z', @@ -38,6 +38,13 @@ void main() { expect(record.progress, 0.25); }); + test('tasks with the ftp protocol decode', () { + final task = buildTask([], 60, protocol: 'ftp'); + expect(task.protocol, api.Protocol.ftp); + final record = TaskRecord.fromApi(task); + expect(record.protocol, api.Protocol.ftp); + }); + test('empty selection preserves default all files', () { final record = TaskRecord.fromApi(buildTask([], 60)); expect(record.files.length, 3); diff --git a/ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart b/ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart index f07f661..fa08a75 100644 --- a/ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart +++ b/ui/flutter/test/features/tasks/presentation/create_task_window_page_test.dart @@ -12,6 +12,7 @@ import 'package:gopeed/api/model/downloader_config.dart'; import 'package:gopeed/api/model/options.dart'; import 'package:gopeed/api/model/request.dart'; import 'package:gopeed/api/model/resolve_result.dart'; +import 'package:gopeed/api/model/resolve_task.dart'; import 'package:gopeed/api/model/resource.dart'; import 'package:gopeed/core/capabilities/app_capabilities.dart'; import 'package:gopeed/core/capabilities/app_navigation_capability.dart'; @@ -611,6 +612,168 @@ void main() { expect(find.byKey(const ValueKey('create-history-clear')), findsOneWidget); expect(find.byKey(const ValueKey('create-history-close')), findsOneWidget); }); + + group('FTP links', _ftpTests); +} + +Future _pumpCreateRoute(WidgetTester tester, CapabilityRegistry registry, {CreateTask? initialTask}) async { + tester.view.physicalSize = const Size(1024, 900); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.resetPhysicalSize); + addTearDown(tester.view.resetDevicePixelRatio); + final router = GoRouter( + initialLocation: '/create', + routes: [ + GoRoute(path: '/', builder: (_, _) => const SizedBox()), + GoRoute( + path: '/create', + builder: (_, _) => CreateTaskWindowPage(initialTask: initialTask), + ), + ], + ); + addTearDown(router.dispose); + await tester.pumpWidget( + ProviderScope( + overrides: [appCapabilitiesProvider.overrideWithValue(AppCapabilities(LocalCapabilityInvoker(registry)))], + child: shad.ShadcnApp.router( + theme: AppTheme.light(), + materialTheme: AppTheme.materialLight(), + builder: (context, child) => AppComponentThemes(child: child ?? const SizedBox.shrink()), + routerConfig: router, + ), + ), + ); + await tester.pumpAndSettle(); +} + +void _ftpTests() { + const urlInput = ValueKey('create-task-url-input'); + const connectionsInput = ValueKey('create-task-connections-input'); + const tlsSelect = ValueKey('create-task-ftp-tls'); + + testWidgets('ftp links offer the TLS mode and carry FTP options through resolve', (tester) async { + ResolveTask? resolved; + CreateTask? submitted; + final registry = CapabilityRegistry(createAppCapabilityCodecs()) + ..bind(GopeedMethods.getConfig, (_) => DownloaderConfig(downloadDir: '/downloads')) + ..bind(GopeedMethods.resolve, (task) { + resolved = task; + return ResolveResult( + id: 'ftp-resolved', + res: Resource( + name: 'big.iso', + files: [FileInfo(name: 'big.iso', size: 1024)], + ), + ); + }) + ..bind(GopeedMethods.createTask, (task) { + submitted = task; + return 'ftp-task'; + }) + ..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit()); + await _pumpCreateRoute(tester, registry); + expect(find.byKey(tlsSelect), findsNothing); + + // ftps:// and ftpes:// fix the TLS mode, so only ftp:// offers the select. + for (final url in ['ftps://files.example/pub/big.iso', 'ftpes://files.example/pub/big.iso']) { + await tester.enterText(find.byKey(urlInput), url); + await tester.pump(); + expect(find.byKey(tlsSelect), findsNothing, reason: url); + } + await tester.enterText(find.byKey(urlInput), 'FTP://files.example/pub/big.iso'); + await tester.pump(); + expect(find.byKey(tlsSelect), findsOneWidget); + expect(find.descendant(of: find.byKey(tlsSelect), matching: find.text('None')), findsOneWidget); + + await tester.tap(find.byKey(tlsSelect)); + await tester.pumpAndSettle(); + expect(find.byKey(const ValueKey('create-task-ftp-tls-implicit')), findsOneWidget); + await tester.tap(find.byKey(const ValueKey('create-task-ftp-tls-explicit'))); + await tester.pumpAndSettle(); + expect(find.descendant(of: find.byKey(tlsSelect), matching: find.text('Explicit TLS (FTPES)')), findsOneWidget); + + await tester.enterText(find.byKey(connectionsInput), '64'); + await tester.pump(); + await tester.tap(find.byKey(const ValueKey('create-task-confirm-button'))); + // The confirm button spins while the resolve dialog is open, so the frames never settle. + await tester.pump(); + await tester.pump(const Duration(milliseconds: 300)); + await tester.tap(find.byKey(const ValueKey('resolve-create-button'))); + await tester.pumpAndSettle(); + + expect(resolved?.req?.url, 'FTP://files.example/pub/big.iso'); + expect(resolved?.req?.extra, isNull); + final resolveOptions = OptsExtraFtp.fromJson(resolved?.opts?.extra! as Map); + expect(resolveOptions.connections, 32); + expect(resolveOptions.tls, 'explicit'); + expect(submitted?.rid, 'ftp-resolved'); + final createOptions = submitted?.opts?.extra! as Map; + expect(createOptions['connections'], 32); + expect(createOptions['tls'], 'explicit'); + }); + + testWidgets('an ftp .torrent link restores its TLS mode and is not treated as BitTorrent', (tester) async { + CreateTask? submitted; + final config = DownloaderConfig(downloadDir: '/downloads')..extra.defaultDirectDownload = true; + final registry = CapabilityRegistry(createAppCapabilityCodecs()) + ..bind(GopeedMethods.getConfig, (_) => config) + ..bind(GopeedMethods.createTask, (task) { + submitted = task; + return 'ftp-task'; + }) + ..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit()); + await _pumpCreateRoute( + tester, + registry, + initialTask: CreateTask( + req: Request(url: 'ftp://files.example/pub/linux.torrent'), + opts: Options(extra: OptsExtraFtp(connections: 6, tls: 'implicit').toJson()), + ), + ); + + expect(_fieldText(tester, 'create-task-connections-input'), '6'); + expect(find.descendant(of: find.byKey(tlsSelect), matching: find.text('Implicit TLS (FTPS)')), findsOneWidget); + await tester.tap(find.text('Advanced')); + await tester.pumpAndSettle(); + // The HTTP tab holds the certificate and post-download switches FTP uses; trackers stay hidden. + expect(find.byKey(const ValueKey('create-task-skip-verify-cert')), findsOneWidget); + expect(find.text('Trackers'), findsNothing); + + await tester.tap(find.text('Confirm')); + await tester.pumpAndSettle(); + expect(submitted?.req?.url, 'ftp://files.example/pub/linux.torrent'); + expect(submitted?.req?.extra, isNull); + final options = OptsExtraFtp.fromJson(submitted?.opts?.extra! as Map); + expect(options.connections, 6); + expect(options.tls, 'implicit'); + }); + + testWidgets('mixed links cap only the FTP task at 32 connections', (tester) async { + final submitted = []; + final registry = CapabilityRegistry(createAppCapabilityCodecs()) + ..bind(GopeedMethods.getConfig, (_) => DownloaderConfig(downloadDir: '/downloads')) + ..bind(GopeedMethods.createTask, (task) { + submitted.add(task); + return 'task-${submitted.length}'; + }) + ..bind(StorageMethods.saveCreateHistory, (_) => const RpcUnit()); + await _pumpCreateRoute(tester, registry); + + await tester.enterText(find.byKey(urlInput), 'https://example.com/a.zip\nftpes://files.example/b.iso'); + await tester.pump(); + expect(find.byKey(tlsSelect), findsNothing); + await tester.enterText(find.byKey(connectionsInput), '64'); + await tester.pump(); + await tester.tap(find.text('Confirm')); + await tester.pumpAndSettle(); + + final byUrl = {for (final task in submitted) task.req!.url: task.opts!.extra! as Map}; + expect(byUrl.keys, unorderedEquals(['https://example.com/a.zip', 'ftpes://files.example/b.iso'])); + expect(byUrl['https://example.com/a.zip']!['connections'], 64); + expect(byUrl['https://example.com/a.zip']!.containsKey('tls'), isFalse); + expect(byUrl['ftpes://files.example/b.iso']!['connections'], 32); + expect(byUrl['ftpes://files.example/b.iso']!.containsKey('tls'), isFalse); + }); } String _fieldText(WidgetTester tester, String key) { -- 2.55.0