diff --git a/.dockerignore b/.dockerignore index 7af87d81..84b6ddf3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ downloads/ data/ cache/ docs/ -config.example.toml docker-compose.* -playwright/ \ No newline at end of file +playwright/ +web/ui/node_modules +web/dist diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7d34bfa1..f9851644 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -14,7 +14,11 @@ concurrency: cancel-in-progress: true jobs: + test: + uses: ./.github/workflows/ci.yml + prepare: + needs: test runs-on: ubuntu-latest permissions: contents: read @@ -47,7 +51,6 @@ jobs: strategy: matrix: arch: [amd64, arm64] - type: [default, micro, pico] fail-fast: false runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} @@ -70,7 +73,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ${{ matrix.type == 'default' && './Dockerfile' || format('./Dockerfile.{0}', matrix.type) }} + file: ./Dockerfile platforms: ${{ matrix.arch == 'amd64' && 'linux/amd64' || 'linux/arm64' }} outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true build-args: | @@ -93,7 +96,7 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v6 with: - name: digest-${{ matrix.type }}-${{ matrix.arch }} + name: digest-${{ matrix.arch }} path: /tmp/digests/digest if-no-files-found: error retention-days: 1 @@ -103,9 +106,6 @@ jobs: runs-on: ubuntu-latest permissions: packages: write - strategy: - matrix: - type: [default, micro, pico] steps: - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -118,43 +118,40 @@ jobs: uses: actions/download-artifact@v7 with: path: /tmp/digests - pattern: digest-${{ matrix.type }}-* + pattern: digest-* merge-multiple: false - - name: Create and push manifest lists + - name: Create and push manifest list run: | REPO="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" VERSION="${{ needs.prepare.outputs.version }}" MAJOR_MINOR="${{ needs.prepare.outputs.major_minor }}" SHA="${{ needs.prepare.outputs.short_sha }}" - TYPE="${{ matrix.type }}" - DIGEST_AMD64=$(cat /tmp/digests/digest-${TYPE}-amd64/digest) - DIGEST_ARM64=$(cat /tmp/digests/digest-${TYPE}-arm64/digest) + DIGEST_AMD64=$(cat /tmp/digests/digest-amd64/digest) + DIGEST_ARM64=$(cat /tmp/digests/digest-arm64/digest) - echo "Found digests for $TYPE:" echo "AMD64: $DIGEST_AMD64" echo "ARM64: $DIGEST_ARM64" - TAGS=() - - if [ "$TYPE" == "default" ]; then - TAGS+=("$REPO:latest") - TAGS+=("$REPO:$VERSION") - TAGS+=("$REPO:$MAJOR_MINOR") - TAGS+=("$REPO:sha-$SHA") - else - TAGS+=("$REPO:$TYPE") - TAGS+=("$REPO:$TYPE-latest") - TAGS+=("$REPO:$TYPE-$VERSION") - fi - SRC_AMD64="${REPO}@${DIGEST_AMD64}" SRC_ARM64="${REPO}@${DIGEST_ARM64}" - echo "Creating manifest list with sources:" - echo " $SRC_AMD64" - echo " $SRC_ARM64" + if [[ "$VERSION" == *-* ]]; then + # Prereleases must not move stable channels such as latest or 1.0. + echo "Prerelease $VERSION: publishing exact and commit tags only" + TAGS=( + "$REPO:$VERSION" + "$REPO:sha-$SHA" + ) + else + TAGS=( + "$REPO:latest" + "$REPO:$VERSION" + "$REPO:$MAJOR_MINOR" + "$REPO:sha-$SHA" + ) + fi for TAG in "${TAGS[@]}"; do echo "Pushing tag: $TAG" diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 77885e9e..57877ccf 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -10,8 +10,12 @@ permissions: packages: write jobs: + test: + uses: ./.github/workflows/ci.yml + create-release: name: Create Release + needs: test runs-on: ubuntu-latest steps: - name: Checkout @@ -21,18 +25,26 @@ jobs: - name: Create Release uses: softprops/action-gh-release@v3 + with: + prerelease: ${{ contains(github.ref_name, '-') }} + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 12.14.0 - name: Setup node uses: actions/setup-node@v7 with: - node-version: 24 + node-version: 26 - - run: npx changelogithub + - run: pnpm dlx changelogithub env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} build-matrix: name: Release Go Binary + needs: test runs-on: ubuntu-latest strategy: matrix: @@ -45,12 +57,35 @@ jobs: - name: Checkout uses: actions/checkout@v6 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.27.x" + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 12.14.0 + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 26 + cache: pnpm + cache-dependency-path: web/ui/pnpm-lock.yaml + - name: Extract version from Git Ref id: extract_version run: | VERSION=$(echo "${{ github.ref }}" | sed 's/refs\/tags\/v//') echo "VERSION=${VERSION}" >> $GITHUB_ENV + - name: Build Web UI + run: | + cd web/ui + pnpm install --frozen-lockfile + pnpm run build + - name: Release Go Binary uses: wangyoucao577/go-release-action@v1 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..9cbc5979 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,100 @@ +name: CI + +on: + push: + branches: + - main + - v1-dev + pull_request: + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend: + name: Backend tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v5 + with: + go-version: "1.27.x" + cache: true + + - name: Test + run: go test ./... + + - name: Vet + run: go vet ./... + + - name: API end-to-end tests + run: go test -tags=e2e ./e2e -count=1 + + - name: Telegram end-to-end tests + run: go test -tags=e2e ./e2e/telegram -count=1 + + web: + name: Web checks + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + + - uses: pnpm/action-setup@v4 + with: + version: 12.14.0 + + - uses: actions/setup-node@v7 + with: + node-version: 26 + cache: pnpm + cache-dependency-path: web/ui/pnpm-lock.yaml + + - name: Install + run: pnpm --dir web/ui install --frozen-lockfile + + - name: Typecheck + run: pnpm --dir web/ui run typecheck + + - name: Build + run: pnpm --dir web/ui run build + + web-e2e: + name: Browser end-to-end tests + needs: web + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v5 + with: + go-version: "1.27.x" + cache: true + + - uses: pnpm/action-setup@v4 + with: + version: 12.14.0 + + - uses: actions/setup-node@v7 + with: + node-version: 26 + cache: pnpm + cache-dependency-path: web/ui/pnpm-lock.yaml + + - name: Install + run: pnpm --dir web/ui install --frozen-lockfile + + - name: Install Chromium + run: pnpm --dir web/ui exec playwright install --with-deps chromium + + - name: Browser tests + run: make e2e-web diff --git a/.gitignore b/.gitignore index 480bc4ef..e69e2aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ testplugins/ tmp-* saveany-bot .codegraph/ +web/ui/node_modules/ +web/dist/* +!web/dist/.gitkeep diff --git a/Dockerfile b/Dockerfile index 4d155aa0..9f1577ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,12 @@ -FROM golang:alpine AS builder +FROM node:26-alpine AS web +WORKDIR /ui +RUN npm install -g pnpm +COPY web/ui/package.json web/ui/pnpm-lock.yaml web/ui/pnpm-workspace.yaml ./ +RUN pnpm install --frozen-lockfile +COPY web/ui ./ +RUN pnpm run build + +FROM golang:1.27-alpine AS builder ARG VERSION="dev" ARG GitCommit="Unknown" @@ -11,6 +19,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ go mod download COPY . . +COPY --from=web /dist ./web/dist RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg \ CGO_ENABLED=0 \ diff --git a/Dockerfile.micro b/Dockerfile.micro deleted file mode 100644 index ca82d24b..00000000 --- a/Dockerfile.micro +++ /dev/null @@ -1,41 +0,0 @@ -FROM golang:alpine AS builder - -ARG VERSION="dev" -ARG GitCommit="Unknown" -ARG BuildTime="Unknown" - -WORKDIR /app - -COPY go.mod go.sum ./ -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -COPY . . -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - CGO_ENABLED=0 \ - go build -trimpath \ - -tags=no_jsparser,no_minio \ - -ldflags=" \ - -s -w \ - -X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \ - -X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \ - -X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \ - -X 'github.com/krau/SaveAny-Bot/config.Docker=true' \ - " \ - -o saveany-bot . - -FROM alpine:latest - -RUN apk add --no-cache curl - -WORKDIR /app - -COPY --from=builder /app/saveany-bot . -COPY entrypoint.sh . - -RUN chmod +x /app/saveany-bot && \ - chmod +x /app/entrypoint.sh - -ENTRYPOINT ["/app/entrypoint.sh"] - diff --git a/Dockerfile.pico b/Dockerfile.pico deleted file mode 100644 index 8671dad8..00000000 --- a/Dockerfile.pico +++ /dev/null @@ -1,41 +0,0 @@ -# pico is the minimum build of SaveAnyBot, which disables all the optional features like JS parsing and MinIO support. -FROM golang:alpine AS builder - -ARG VERSION="dev" -ARG GitCommit="Unknown" -ARG BuildTime="Unknown" - -WORKDIR /app - -RUN apk add --no-cache ca-certificates - -COPY go.mod go.sum ./ -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -COPY . . -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg \ - CGO_ENABLED=0 \ - go build -trimpath \ - -tags=no_jsparser,no_minio,sqlite_glebarez \ - -ldflags=" \ - -s -w \ - -X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \ - -X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \ - -X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \ - -X 'github.com/krau/SaveAny-Bot/config.Docker=true' \ - " \ - -o saveany-bot . && chmod +x saveany-bot - -FROM scratch - -WORKDIR /app - -COPY --from=builder /app/saveany-bot . -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -ENV SSL_CERT_DIR=/etc/ssl/certs - -ENTRYPOINT ["/app/saveany-bot"] diff --git a/Makefile b/Makefile index 404f3819..3c21fbe7 100644 --- a/Makefile +++ b/Makefile @@ -6,3 +6,28 @@ run: down: docker compose -f docker-compose.local.yml down --remove-orphans + +web: + cd web/ui && pnpm install --frozen-lockfile && pnpm run build && touch ../dist/.gitkeep + +build: web + CGO_ENABLED=0 go build -trimpath -o saveany-bot . + +testenv: + go run . testenv + +e2e: e2e-api e2e-telegram e2e-web + +e2e-api: + go test -tags=e2e ./e2e -count=1 + +e2e-telegram: + go test -tags=e2e ./e2e/telegram -count=1 + +e2e-web-setup: + pnpm --dir web/ui exec playwright install chromium + +e2e-web: web + mkdir -p tmp + go build -o tmp/saveany-bot-e2e . + pnpm --dir web/ui exec playwright test diff --git a/README.md b/README.md index 1ee7dc56..ca01365a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-# Save Any Bot +# Save Any Bot **English** | [简体中文](./README_zh.md) @@ -40,39 +40,45 @@ ## 📦 Quick Start -Create a `config.toml` file with the following content: - -```toml -lang = "en" # Language setting, "en" for English -[telegram] -token = "" # Your bot token, obtained from @BotFather -[telegram.proxy] -# Enable proxy for Telegram -enable = false -url = "socks5://127.0.0.1:7890" - -[[storages]] -name = "Local Disk" -type = "local" -enable = true -base_path = "./downloads" - -[[users]] -id = 114514 # Your Telegram account id -storages = [] -blacklist = true -``` - -Run Save Any Bot with Docker: +SaveAny-Bot is configured in the web console. Start it with empty data, downloads, and cache directories: ```bash docker run -d --name saveany-bot \ - -v ./config.toml:/app/config.toml \ + -p 28080:28080 \ + -e SAVEANY_LISTEN=0.0.0.0:28080 \ + -v ./data:/app/data \ -v ./downloads:/app/downloads \ + -v ./cache:/app/cache \ ghcr.io/krau/saveany-bot:latest ``` -Please [**read the docs**](https://sabot.unv.app/en/) for more configuration options and usage. +The web UI listens on `127.0.0.1:28080` by default. The startup log prints a direct URL with an access token: + +```bash +docker logs saveany-bot | grep -E "Web UI available at|Generated web admin password" +``` + +For a binary deployment, pass the address directly: + +```bash +./saveany-bot --listen 0.0.0.0:28080 +``` + +Open the URL to sign in. On the first start, the same log also contains the generated admin password; use it to change the password from the sidebar after signing in. Unless an access token is set under **Settings → Web**, a new token is generated on every start. The Guide page covers the required setup: + +1. Connect your Telegram bot. +2. Add a storage destination. +3. Allow your Telegram account. + +Then send files or message links to the bot and watch them in the Tasks page. + +To run the binary, download a release, extract it, and start it. It creates `data/` and prints the same tokenized URL: + +```bash +./saveany-bot +``` + +Please [**read the docs**](https://sabot.unv.app/en/) for storage backends and usage. ## Sponsors diff --git a/README_zh.md b/README_zh.md index 8907c82e..49e952cd 100644 --- a/README_zh.md +++ b/README_zh.md @@ -1,6 +1,6 @@
-# Save Any Bot +# Save Any Bot > **把 Telegram 上的文件转存到多种存储端** @@ -15,19 +15,19 @@
-## 🎯 特性 +## 特性 -- 支持文档/视频/图片/贴纸…甚至还有 [Telegraph](https://telegra.ph/) -- 破解禁止保存的文件 +- 支持文档、视频、图片、贴纸及 [Telegraph](https://telegra.ph/) 图集 +- 支持转存受限频道中禁止转发与保存的内容 - 批量下载 - 流式传输 -- 多用户使用 -- 基于存储规则的自动整理 -- 监听并自动转存指定聊天的消息, 支持过滤 -- 在不同存储端之间转存文件 -- 集成 yt-dlp, 从所支持的网站下载并转存媒体文件 -- 集成 Aria2, 支持直链/磁力下载和转存 -- 使用 js 编写解析器插件以转存任意网站的文件 +- 多用户支持 +- 基于规则自动分类整理 +- 监听指定聊天并按过滤器自动转存 +- 存储端之间互相传输文件 +- 集成 yt-dlp,下载视频并转存 +- 集成 Aria2,支持直链与磁力链接转存 +- 支持编写 JavaScript 解析器插件扩展站点 - 存储端支持: - Alist - S3 @@ -38,38 +38,45 @@ ## 快速开始 -创建文件 `config.toml` 并填入以下内容: - -```toml -[telegram] -token = "" # 你的 Bot Token, 在 @BotFather 获取 -[telegram.proxy] -# 启用代理连接 telegram -enable = false -url = "socks5://127.0.0.1:7890" - -[[storages]] -name = "本地磁盘" -type = "local" -enable = true -base_path = "./downloads" - -[[users]] -id = 114514 # 你的 Telegram 账号 id -storages = [] -blacklist = true -``` - -使用 Docker 运行 Save Any Bot: +SaveAny-Bot 通过 Web 控制台配置。启动时挂载数据、下载和缓存目录: ```bash docker run -d --name saveany-bot \ - -v ./config.toml:/app/config.toml \ + -p 28080:28080 \ + -e SAVEANY_LISTEN=0.0.0.0:28080 \ + -v ./data:/app/data \ -v ./downloads:/app/downloads \ + -v ./cache:/app/cache \ ghcr.io/krau/saveany-bot:latest ``` -请 [**查看文档**](https://sabot.unv.app/) 以获取更多配置选项和使用方法. +Web 界面默认监听 `127.0.0.1:28080`。启动日志会输出带访问 Token 的直达地址: + +```bash +docker logs saveany-bot | grep -E "Web UI available at|Generated web admin password" +``` + +直接运行二进制文件时,可通过命令行指定监听地址: + +```bash +./saveany-bot --listen 0.0.0.0:28080 +``` + +访问该地址即可登录。首启日志同时包含初始管理员密码,登录后可在侧栏修改。若未在“设置 → Web”中指定 Token,每次启动均会重新生成。初次登录可根据指南页面完成配置: + +1. 连接 Telegram Bot。 +2. 添加存储位置。 +3. 授权 Telegram 账号。 + +完成后向 Bot 发送文件或消息链接,即可在任务页查看转存进度。 + +若使用预编译二进制文件,解压后直接运行,程序将自动创建 `data/` 目录并输出访问地址: + +```bash +./saveany-bot +``` + +更多存储配置与使用方法请参阅[官方文档](https://sabot.unv.app/)。 ## 赞助 diff --git a/api/auth.go b/api/auth.go index 454ed055..2100124d 100644 --- a/api/auth.go +++ b/api/auth.go @@ -1,38 +1,283 @@ package api import ( + "context" + "crypto/rand" "crypto/subtle" + "encoding/base64" + "encoding/json" + "net" "net/http" "strings" + "sync" + "time" - "github.com/krau/SaveAny-Bot/config" + "github.com/charmbracelet/log" + "golang.org/x/time/rate" + + "github.com/krau/SaveAny-Bot/database" ) -func AuthMiddleware() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cfg := config.C().API +const ( + sessionCookieName = "sab_session" + sessionTTL = 7 * 24 * time.Hour + csrfHeader = "X-CSRF-Token" +) - authHeader := r.Header.Get("Authorization") - if authHeader == "" { - WriteError(w, http.StatusUnauthorized, "unauthorized", "missing authorization header") - return +type sessionCtxKey struct{} + +func sessionFromContext(ctx context.Context) *database.WebSession { + session, _ := ctx.Value(sessionCtxKey{}).(*database.WebSession) + return session +} + +type loginLimiter struct { + mu sync.Mutex + limiter map[string]*rate.Limiter + lastSeen map[string]time.Time +} + +func newLoginLimiter() *loginLimiter { + return &loginLimiter{ + limiter: make(map[string]*rate.Limiter), + lastSeen: make(map[string]time.Time), + } +} + +func (l *loginLimiter) cleanup(maxAge time.Duration) { + cutoff := time.Now().Add(-maxAge) + l.mu.Lock() + defer l.mu.Unlock() + for key, seen := range l.lastSeen { + if seen.Before(cutoff) { + delete(l.lastSeen, key) + delete(l.limiter, key) + } + } +} + +func (l *loginLimiter) allow(key string) bool { + l.mu.Lock() + defer l.mu.Unlock() + l.lastSeen[key] = time.Now() + lim, ok := l.limiter[key] + if !ok { + lim = rate.NewLimiter(rate.Every(12*time.Second), 5) + l.limiter[key] = lim + } + return lim.Allow() +} + +func randomToken() (string, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} + +func (s *server) isPublic(r *http.Request) bool { + if r.URL.Path == "/health" { + return true + } + if r.Method == http.MethodGet && r.URL.Path == "/api/v1/auth/token" { + return true + } + return r.Method == http.MethodPost && r.URL.Path == "/api/v1/auth/login" +} + +func (s *server) authToken() string { + if token := strings.TrimSpace(s.deps.Settings.Config().Web.APIToken); token != "" { + return token + } + return s.deps.WebToken +} + +func (s *server) validToken(token string) bool { + expected := s.authToken() + if expected == "" || token == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1 +} + +func (s *server) authMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.isPublic(r) { + next.ServeHTTP(w, r) + return + } + + if token := s.authToken(); token != "" { + header := r.Header.Get("Authorization") + if len(header) > 7 && strings.EqualFold(header[:7], "Bearer ") { + if subtle.ConstantTimeCompare([]byte(header[7:]), []byte(token)) == 1 { + next.ServeHTTP(w, r) + return + } } + } - parts := strings.SplitN(authHeader, " ", 2) - if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" { - WriteError(w, http.StatusUnauthorized, "unauthorized", "invalid authorization header format") + cookie, err := r.Cookie(sessionCookieName) + if err != nil || cookie.Value == "" { + WriteError(w, http.StatusUnauthorized, "unauthorized", "authentication required") + return + } + session, err := database.GetWebSession(r.Context(), cookie.Value) + if err != nil { + WriteError(w, http.StatusUnauthorized, "unauthorized", "session expired") + return + } + if r.Method != http.MethodGet && r.Method != http.MethodHead && r.Method != http.MethodOptions { + if subtle.ConstantTimeCompare([]byte(r.Header.Get(csrfHeader)), []byte(session.CSRF)) != 1 { + WriteError(w, http.StatusForbidden, "csrf_failed", "invalid csrf token") return } + } + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), sessionCtxKey{}, session))) + }) +} - token := parts[1] +func (s *server) tokenLogin(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if !s.validToken(strings.TrimSpace(r.URL.Query().Get("token"))) { + WriteError(w, http.StatusUnauthorized, "invalid_token", "invalid access token") + return + } + if _, err := s.establishSession(w, r); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", "failed to create session") + return + } + http.Redirect(w, r, "/", http.StatusSeeOther) +} - if subtle.ConstantTimeCompare([]byte(token), []byte(cfg.Token)) != 1 { - WriteError(w, http.StatusUnauthorized, "unauthorized", "invalid token") - return - } +func (s *server) login(w http.ResponseWriter, r *http.Request) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + if !s.logins.allow(host) { + WriteError(w, http.StatusTooManyRequests, "too_many_attempts", "too many login attempts") + return + } + var body struct { + Password string `json:"password"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if !s.deps.Settings.VerifyAdminPassword(body.Password) { + WriteError(w, http.StatusUnauthorized, "invalid_credentials", "invalid password") + return + } + csrf, err := s.establishSession(w, r) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", "failed to create session") + return + } + WriteJSON(w, http.StatusOK, map[string]string{"csrf_token": csrf}) +} - next.ServeHTTP(w, r) - }) +func (s *server) establishSession(w http.ResponseWriter, r *http.Request) (string, error) { + id, err := randomToken() + if err != nil { + return "", err + } + csrf, err := randomToken() + if err != nil { + return "", err + } + session := &database.WebSession{ + ID: id, + CSRF: csrf, + CreatedAt: time.Now(), + ExpiresAt: time.Now().Add(sessionTTL), + } + if err := database.CreateWebSession(r.Context(), session); err != nil { + return "", err + } + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, + Value: id, + Path: "/", + MaxAge: int(sessionTTL.Seconds()), + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + Secure: s.deps.Settings.Config().Web.SecureCookie, + }) + return csrf, nil +} + +func (s *server) logout(w http.ResponseWriter, r *http.Request) { + if session := sessionFromContext(r.Context()); session != nil { + if err := database.DeleteWebSession(r.Context(), session.ID); err != nil { + log.FromContext(r.Context()).Warnf("Failed to delete web session: %v", err) + } + } + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, + Value: "", + Path: "/", + MaxAge: -1, + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + }) + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) me(w http.ResponseWriter, r *http.Request) { + session := sessionFromContext(r.Context()) + if session == nil { + WriteJSON(w, http.StatusOK, map[string]any{"authenticated": false}) + return + } + WriteJSON(w, http.StatusOK, map[string]any{ + "authenticated": true, + "csrf_token": session.CSRF, + }) +} + +func (s *server) changePassword(w http.ResponseWriter, r *http.Request) { + var body struct { + CurrentPassword string `json:"current_password"` + NewPassword string `json:"new_password"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if err := s.deps.Settings.ChangeAdminPassword(body.CurrentPassword, body.NewPassword); err != nil { + WriteError(w, http.StatusBadRequest, "password_change_failed", err.Error()) + return + } + keepID := "" + if session := sessionFromContext(r.Context()); session != nil { + keepID = session.ID + } + if err := database.DeleteOtherWebSessions(r.Context(), keepID); err != nil { + WriteError(w, http.StatusInternalServerError, "session_cleanup_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +func (s *server) sessionCleanup(ctx context.Context) { + if err := database.DeleteExpiredWebSessions(ctx); err != nil { + log.FromContext(ctx).Warnf("Failed to delete expired web sessions: %v", err) + } + s.logins.cleanup(24 * time.Hour) + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := database.DeleteExpiredWebSessions(ctx); err != nil { + log.FromContext(ctx).Warnf("Failed to delete expired web sessions: %v", err) + } + s.logins.cleanup(24 * time.Hour) + } } } diff --git a/api/factory.go b/api/factory.go index f5d511b6..5f160838 100644 --- a/api/factory.go +++ b/api/factory.go @@ -3,9 +3,15 @@ package api import ( "context" "encoding/json" + "errors" "fmt" + "path" "time" + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/client/bot" + userclient "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/core/tasks/aria2dl" @@ -18,14 +24,18 @@ import ( "github.com/krau/SaveAny-Bot/core/tasks/ytdlp" "github.com/krau/SaveAny-Bot/parsers/parsers" "github.com/krau/SaveAny-Bot/pkg/aria2" + storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/parser" "github.com/krau/SaveAny-Bot/pkg/taskevent" "github.com/krau/SaveAny-Bot/pkg/telegraph" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" "github.com/rs/xid" ) +var ErrClientUnavailable = errors.New("no telegram client available") + type TaskFactory struct { ctx context.Context } @@ -35,32 +45,47 @@ func NewTaskFactory(ctx context.Context) *TaskFactory { } func (f *TaskFactory) CreateTask(req *CreateTaskRequest) (*CreateTaskResponse, error) { - stor, ok := storage.GetStorage(req.Storage) - if !ok { - return nil, fmt.Errorf("storage not found: %s", req.Storage) + if req.StorageID == 0 { + return nil, errors.New("storage_id is required") + } + stor, err := storage.GetStorageByID(f.ctx, req.StorageID) + if err != nil { + return nil, fmt.Errorf("storage_id %d not found: %w", req.StorageID, err) + } + if req.Type != tasktype.TaskTypeTransfer && stor.Type() == storenum.Telegram && bot.ExtContext() == nil && userclient.GetCtx() == nil { + return nil, ErrClientUnavailable } taskID := xid.New().String() createdAt := time.Now() + var resp *CreateTaskResponse switch req.Type { case tasktype.TaskTypeDirectlinks: - return f.createDirectLinksTask(taskID, createdAt, req, stor) + resp, err = f.createDirectLinksTask(taskID, createdAt, req, stor) case tasktype.TaskTypeYtdlp: - return f.createYTDLPTask(taskID, createdAt, req, stor) + resp, err = f.createYTDLPTask(taskID, createdAt, req, stor) case tasktype.TaskTypeAria2: - return f.createAria2Task(taskID, createdAt, req, stor) + resp, err = f.createAria2Task(taskID, createdAt, req, stor) case tasktype.TaskTypeParseditem: - return f.createParsedTask(taskID, createdAt, req, stor) + resp, err = f.createParsedTask(taskID, createdAt, req, stor) case tasktype.TaskTypeTgfiles: - return f.createTGFilesTask(taskID, createdAt, req, stor) + resp, err = f.createTGFilesTask(taskID, createdAt, req, stor) case tasktype.TaskTypeTphpics: - return f.createTPHPicsTask(taskID, createdAt, req, stor) + resp, err = f.createTPHPicsTask(taskID, createdAt, req, stor) case tasktype.TaskTypeTransfer: - return f.createTransferTask(taskID, createdAt, req) + resp, err = f.createTransferTask(taskID, createdAt, req) default: return nil, fmt.Errorf("unsupported task type: %s", req.Type) } + if err != nil { + return nil, err + } + if req.Type != tasktype.TaskTypeTransfer { + resp.StorageID = storage.ID(stor) + resp.StorageName = stor.Name() + } + return resp, nil } func (f *TaskFactory) registerAndEnqueueTask(task core.Executable, taskType tasktype.TaskType, storageName, path, webhook string) error { @@ -71,12 +96,19 @@ func (f *TaskFactory) registerAndEnqueueTask(task core.Executable, taskType task // the API store (and fire the webhook on terminal states) without the task // knowing about the API. taskCtx := taskevent.WithSink(f.ctx, info) + if botCtx := bot.ExtContext(); botCtx != nil { + taskCtx = tgutil.ExtWithContext(taskCtx, botCtx) + } else if userCtx := userclient.GetCtx(); userCtx != nil { + taskCtx = tgutil.ExtWithContext(taskCtx, userCtx) + } - err := core.AddTask(taskCtx, task) - if err != nil { + if err := core.AddTaskWithSource(taskCtx, task, core.TaskSourceAPI); err != nil { DeleteTask(taskID) return fmt.Errorf("failed to add task: %w", err) } + if err := core.EnrichTaskRecord(f.ctx, taskID, storageName, path); err != nil { + log.Errorf("Failed to enrich task record %s: %v", taskID, err) + } return nil } @@ -93,16 +125,18 @@ func (f *TaskFactory) createDirectLinksTask(taskID string, createdAt time.Time, task := directlinks.NewTask(taskID, f.ctx, params.URLs, stor, req.Path, nil) - err := f.registerAndEnqueueTask(task, tasktype.TaskTypeDirectlinks, req.Storage, req.Path, req.Webhook) + err := f.registerAndEnqueueTask(task, tasktype.TaskTypeDirectlinks, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeDirectlinks, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeDirectlinks, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -118,16 +152,18 @@ func (f *TaskFactory) createYTDLPTask(taskID string, createdAt time.Time, req *C task := ytdlp.NewTask(taskID, f.ctx, params.URLs, params.Flags, stor, req.Path, nil) - err := f.registerAndEnqueueTask(task, tasktype.TaskTypeYtdlp, req.Storage, req.Path, req.Webhook) + err := f.registerAndEnqueueTask(task, tasktype.TaskTypeYtdlp, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeYtdlp, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeYtdlp, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -158,16 +194,18 @@ func (f *TaskFactory) createAria2Task(taskID string, createdAt time.Time, req *C task := aria2dl.NewTask(taskID, f.ctx, gid, params.URLs, aria2Client, stor, req.Path, nil) - err = f.registerAndEnqueueTask(task, tasktype.TaskTypeAria2, req.Storage, req.Path, req.Webhook) + err = f.registerAndEnqueueTask(task, tasktype.TaskTypeAria2, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeAria2, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeAria2, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -200,16 +238,18 @@ func (f *TaskFactory) createParsedTask(taskID string, createdAt time.Time, req * task := parsed.NewTask(taskID, f.ctx, stor, req.Path, item, nil) - err = f.registerAndEnqueueTask(task, tasktype.TaskTypeParseditem, req.Storage, req.Path, req.Webhook) + err = f.registerAndEnqueueTask(task, tasktype.TaskTypeParseditem, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeParseditem, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeParseditem, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -219,23 +259,28 @@ func (f *TaskFactory) createTGFilesTask(taskID string, createdAt time.Time, req return nil, fmt.Errorf("invalid params: %w", err) } - if len(params.MessageLinks) == 0 { - return nil, fmt.Errorf("no message links provided") + if bot.ExtContext() == nil && userclient.GetCtx() == nil { + return nil, ErrClientUnavailable } - files, err := ExtractFilesFromLinks(f.ctx, params.MessageLinks) + var files []tfilepkg.TGFileMessage + var err error + if len(params.MessageLinks) > 0 { + files, err = ExtractFilesFromLinks(f.ctx, params.MessageLinks) + } else { + files, err = ExtractFilesFromRange(f.ctx, params.ChatID, params.MessageRange, params.Filter) + } if err != nil { return nil, fmt.Errorf("failed to extract files: %w", err) } - if len(files) == 0 { - return nil, fmt.Errorf("no files found in provided links") + return nil, fmt.Errorf("no files found") } var task core.Executable if len(files) == 1 { - tfileTask, err := tfile.NewTGFileTask(taskID, f.ctx, files[0], stor, req.Path, nil) + tfileTask, err := tfile.NewTGFileTask(taskID, f.ctx, files[0], stor, path.Join(req.Path, files[0].Name()), nil) if err != nil { return nil, fmt.Errorf("failed to create tfile task: %w", err) } @@ -243,7 +288,7 @@ func (f *TaskFactory) createTGFilesTask(taskID string, createdAt time.Time, req } else { elems := make([]batchtfile.TaskElement, 0, len(files)) for _, file := range files { - elem, err := batchtfile.NewTaskElement(stor, req.Path, file) + elem, err := batchtfile.NewTaskElement(stor, path.Join(req.Path, file.Name()), file) if err != nil { return nil, fmt.Errorf("failed to create task element: %w", err) } @@ -253,16 +298,18 @@ func (f *TaskFactory) createTGFilesTask(taskID string, createdAt time.Time, req task = batchtfile.NewBatchTGFileTask(taskID, f.ctx, elems, nil, true) } - err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTgfiles, req.Storage, req.Path, req.Webhook) + err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTgfiles, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeTgfiles, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeTgfiles, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -288,16 +335,18 @@ func (f *TaskFactory) createTPHPicsTask(taskID string, createdAt time.Time, req client := telegraph.NewClient() task := tphtask.NewTask(taskID, f.ctx, phPath, pics, stor, req.Path, client, nil) - err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTphpics, req.Storage, req.Path, req.Webhook) + err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTphpics, stor.Name(), req.Path, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeTphpics, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeTphpics, + Status: TaskStatusQueued, + StorageID: storage.ID(stor), + StorageName: stor.Name(), + CreatedAt: createdAt, }, nil } @@ -307,24 +356,30 @@ func (f *TaskFactory) createTransferTask(taskID string, createdAt time.Time, req return nil, fmt.Errorf("invalid params: %w", err) } - sourceStor, ok := storage.GetStorage(params.SourceStorage) - if !ok { - return nil, fmt.Errorf("source storage not found: %s", params.SourceStorage) + if params.SourceStorageID == 0 || params.TargetStorageID == 0 { + return nil, errors.New("source_storage_id and target_storage_id are required") + } + sourceStor, err := storage.GetStorageByID(f.ctx, params.SourceStorageID) + if err != nil { + return nil, fmt.Errorf("source_storage_id %d not found: %w", params.SourceStorageID, err) } - targetStor, ok := storage.GetStorage(params.TargetStorage) - if !ok { - return nil, fmt.Errorf("target storage not found: %s", params.TargetStorage) + targetStor, err := storage.GetStorageByID(f.ctx, params.TargetStorageID) + if err != nil { + return nil, fmt.Errorf("target_storage_id %d not found: %w", params.TargetStorageID, err) + } + if targetStor.Type() == storenum.Telegram && bot.ExtContext() == nil && userclient.GetCtx() == nil { + return nil, ErrClientUnavailable } sourceReadable, ok := sourceStor.(storage.StorageReadable) if !ok { - return nil, fmt.Errorf("source storage does not support reading: %s", params.SourceStorage) + return nil, fmt.Errorf("source storage does not support reading: %s", sourceStor.Name()) } sourceListable, ok := sourceStor.(storage.StorageListable) if !ok { - return nil, fmt.Errorf("source storage does not support listing: %s", params.SourceStorage) + return nil, fmt.Errorf("source storage does not support listing: %s", sourceStor.Name()) } files, err := sourceListable.ListFiles(f.ctx, params.SourcePath) @@ -344,15 +399,17 @@ func (f *TaskFactory) createTransferTask(taskID string, createdAt time.Time, req task := transfer.NewTransferTask(taskID, f.ctx, elems, nil, true) - err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTransfer, params.TargetStorage, params.TargetPath, req.Webhook) + err = f.registerAndEnqueueTask(task, tasktype.TaskTypeTransfer, targetStor.Name(), params.TargetPath, req.Webhook) if err != nil { return nil, err } return &CreateTaskResponse{ - TaskID: taskID, - Type: tasktype.TaskTypeTransfer, - Status: TaskStatusQueued, - CreatedAt: createdAt, + TaskID: taskID, + Type: tasktype.TaskTypeTransfer, + Status: TaskStatusQueued, + StorageID: params.TargetStorageID, + StorageName: targetStor.Name(), + CreatedAt: createdAt, }, nil } diff --git a/api/filesystem.go b/api/filesystem.go new file mode 100644 index 00000000..94826ae7 --- /dev/null +++ b/api/filesystem.go @@ -0,0 +1,141 @@ +package api + +import ( + "net/http" + "os" + "path/filepath" + "sort" + "strings" +) + +const pathListLimit = 500 + +type filesystemEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size,omitempty"` +} + +type filesystemListing struct { + Path string `json:"path"` + Parent string `json:"parent,omitempty"` + Root string `json:"root"` + Cwd string `json:"cwd"` + Selected string `json:"selected,omitempty"` + Entries []filesystemEntry `json:"entries"` + Truncated bool `json:"truncated,omitempty"` +} + +// listFilesystem lists one directory on the deployment host for the server-side +// path picker. It returns names and metadata only, never file contents. +func (s *server) listFilesystem(w http.ResponseWriter, r *http.Request) { + requested := strings.TrimSpace(r.URL.Query().Get("path")) + if requested == "" { + cwd, err := os.Getwd() + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + requested = cwd + } + absolute, err := filepath.Abs(requested) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_path", err.Error()) + return + } + dir := absolute + selected := "" + info, err := os.Stat(absolute) + switch { + case err == nil: + if !info.IsDir() { + selected = absolute + dir = filepath.Dir(absolute) + } + case os.IsNotExist(err): + // A target file or directory may not exist yet. Start the picker at the + // nearest existing ancestor instead of failing outright. + for { + parent := filepath.Dir(dir) + if parent == dir { + break + } + dir = parent + if parentInfo, statErr := os.Stat(dir); statErr == nil && parentInfo.IsDir() { + break + } + } + if _, statErr := os.Stat(dir); statErr != nil { + WriteError(w, http.StatusNotFound, "path_not_found", statErr.Error()) + return + } + case os.IsPermission(err): + WriteError(w, http.StatusForbidden, "path_forbidden", err.Error()) + return + default: + WriteError(w, http.StatusBadRequest, "path_unavailable", err.Error()) + return + } + + dirEntries, err := os.ReadDir(dir) + if err != nil { + status := http.StatusBadRequest + code := "path_unavailable" + if os.IsPermission(err) { + status = http.StatusForbidden + code = "path_forbidden" + } + WriteError(w, status, code, err.Error()) + return + } + + entries := make([]filesystemEntry, 0, len(dirEntries)) + for _, entry := range dirEntries { + entryInfo, err := entry.Info() + if err != nil { + continue + } + entries = append(entries, filesystemEntry{ + Name: entry.Name(), + Path: filepath.Join(dir, entry.Name()), + IsDir: entryInfo.IsDir(), + Size: entryInfo.Size(), + }) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDir != entries[j].IsDir { + return entries[i].IsDir + } + return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) + }) + truncated := false + if len(entries) > pathListLimit { + entries = entries[:pathListLimit] + truncated = true + } + + parent := filepath.Dir(dir) + if parent == dir { + parent = "" + } + root := string(filepath.Separator) + if volume := filepath.VolumeName(dir); volume != "" { + root = volume + string(filepath.Separator) + } + cwd, err := os.Getwd() + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + w.Header().Set("Cache-Control", "no-store") + WriteJSON(w, http.StatusOK, filesystemListing{ + Path: dir, + Parent: parent, + Root: root, + Cwd: cwd, + Selected: selected, + Entries: entries, + Truncated: truncated, + }) +} diff --git a/api/handlers.go b/api/handlers.go deleted file mode 100644 index 350196f7..00000000 --- a/api/handlers.go +++ /dev/null @@ -1,270 +0,0 @@ -package api - -import ( - "context" - "encoding/json" - "net/http" - "sort" - "strings" - "time" - - "github.com/krau/SaveAny-Bot/core" - "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" - "github.com/krau/SaveAny-Bot/storage" -) - -type mediaMetadataExtractor func(ctx context.Context, url string) (*MediaMetadataResponse, error) - -type Handlers struct { - factory *TaskFactory - mediaMetadataExtractor mediaMetadataExtractor -} - -func NewHandlers(factory *TaskFactory) *Handlers { - return &Handlers{ - factory: factory, - mediaMetadataExtractor: extractMediaMetadata, - } -} - -func (h *Handlers) CreateTaskHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only POST method is allowed") - return - } - - var req CreateTaskRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - WriteError(w, http.StatusBadRequest, "invalid_request", "failed to decode request body: "+err.Error()) - return - } - - if req.Type == "" { - WriteError(w, http.StatusBadRequest, "invalid_request", "task type is required") - return - } - - if req.Storage == "" { - WriteError(w, http.StatusBadRequest, "invalid_request", "storage is required") - return - } - - resp, err := h.factory.CreateTask(&req) - if err != nil { - WriteError(w, http.StatusBadRequest, "task_creation_failed", err.Error()) - return - } - - WriteJSON(w, http.StatusCreated, resp) -} - -func (h *Handlers) ListTasksHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET method is allowed") - return - } - - tasks := GetAllTasks() - response := make([]TaskInfoResponse, 0, len(tasks)) - - for _, task := range tasks { - info := convertTaskProgressToResponse(task) - response = append(response, info) - } - - // GetAllTasks walks a map, so the order is otherwise nondeterministic. - // Newest first, breaking ties on task ID to keep the order stable across - // calls when two tasks share a creation timestamp. - sort.Slice(response, func(i, j int) bool { - if !response[i].CreatedAt.Equal(response[j].CreatedAt) { - return response[i].CreatedAt.After(response[j].CreatedAt) - } - return response[i].TaskID > response[j].TaskID - }) - - WriteJSON(w, http.StatusOK, TasksListResponse{ - Tasks: response, - Total: len(response), - }) -} - -func (h *Handlers) GetTaskHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET method is allowed") - return - } - - taskID := extractTaskIDFromPath(r.URL.Path) - if taskID == "" { - WriteError(w, http.StatusBadRequest, "invalid_request", "task ID is required") - return - } - - task, ok := GetTask(taskID) - if !ok { - WriteError(w, http.StatusNotFound, "task_not_found", "task not found: "+taskID) - return - } - - resp := convertTaskProgressToResponse(task) - WriteJSON(w, http.StatusOK, resp) -} - -func (h *Handlers) CancelTaskHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only DELETE method is allowed") - return - } - - taskID := extractTaskIDFromPath(r.URL.Path) - if taskID == "" { - WriteError(w, http.StatusBadRequest, "invalid_request", "task ID is required") - return - } - - task, ok := GetTask(taskID) - if !ok { - WriteError(w, http.StatusNotFound, "task_not_found", "task not found: "+taskID) - return - } - - // Cancel the task; the terminal status is set via the task event stream. - if err := core.CancelTask(r.Context(), taskID); err != nil { - WriteError(w, http.StatusInternalServerError, "cancel_failed", "failed to cancel task: "+err.Error()) - return - } - - task.UpdateStatus(TaskStatusCancelled) - WriteJSON(w, http.StatusOK, map[string]string{"message": "task cancelled successfully"}) -} - -func (h *Handlers) ListStoragesHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET method is allowed") - return - } - - all := storage.AllStorages() - storages := make([]StorageInfo, 0, len(all)) - for name, stor := range all { - storages = append(storages, StorageInfo{ - Name: name, - Type: string(stor.Type()), - }) - } - - WriteJSON(w, http.StatusOK, StoragesResponse{Storages: storages}) -} - -func (h *Handlers) GetMediaMetadataHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET method is allowed") - return - } - - mediaURL := strings.TrimSpace(r.URL.Query().Get("url")) - if mediaURL == "" { - WriteError(w, http.StatusBadRequest, "invalid_request", "url query parameter is required") - return - } - - ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) - defer cancel() - - resp, err := h.mediaMetadataExtractor(ctx, mediaURL) - if err != nil { - WriteError(w, http.StatusBadRequest, "metadata_extraction_failed", err.Error()) - return - } - - WriteJSON(w, http.StatusOK, resp) -} - -func (h *Handlers) GetTaskTypesHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET method is allowed") - return - } - - types := []tasktype.TaskType{ - tasktype.TaskTypeDirectlinks, - tasktype.TaskTypeYtdlp, - tasktype.TaskTypeAria2, - tasktype.TaskTypeParseditem, - tasktype.TaskTypeTgfiles, - tasktype.TaskTypeTphpics, - tasktype.TaskTypeTransfer, - } - - WriteJSON(w, http.StatusOK, map[string]any{ - "types": types, - }) -} - -func (h *Handlers) HealthCheckHandler(w http.ResponseWriter, r *http.Request) { - WriteJSON(w, http.StatusOK, map[string]string{ - "status": "ok", - }) -} - -// 路径格式: /api/v1/tasks/:id -func extractTaskIDFromPath(path string) string { - parts := strings.Split(strings.Trim(path, "/"), "/") - if len(parts) < 4 { - return "" - } - return parts[3] -} - -// convertTaskProgressToResponse renders a task's current state, computing -// percent and speed from the snapshot taken under the task's mutex. -func convertTaskProgressToResponse(task *TaskProgressInfo) TaskInfoResponse { - status, total, downloaded, totalFiles, downloadedFiles, startedAt, errMsg, updatedAt := task.snapshot() - - resp := TaskInfoResponse{ - TaskID: task.TaskID, - Type: tasktype.TaskType(task.Type), - Status: status, - Title: task.Title, - Storage: task.Storage, - Path: task.Path, - Error: errMsg, - CreatedAt: task.CreatedAt, - UpdatedAt: updatedAt, - } - - var percent float64 - var speedMBPS float64 - if total > 0 { - percent = float64(downloaded) * 100 / float64(total) - } else if totalFiles > 0 { - percent = float64(downloadedFiles) * 100 / float64(totalFiles) - } - if !startedAt.IsZero() { - elapsed := time.Since(startedAt).Seconds() - if elapsed > 0 { - speedMBPS = float64(downloaded) / elapsed / (1024 * 1024) - } - } - - if total > 0 || totalFiles > 0 { - resp.Progress = &TaskProgress{ - TotalBytes: total, - DownloadedBytes: downloaded, - TotalFiles: totalFiles, - DownloadedFiles: downloadedFiles, - Percent: percent, - SpeedMBPS: speedMBPS, - } - } - - return resp -} - -func NotFoundHandler(w http.ResponseWriter, r *http.Request) { - WriteError(w, http.StatusNotFound, "not_found", "endpoint not found: "+r.URL.Path) -} - -func MethodNotAllowedHandler(w http.ResponseWriter, r *http.Request) { - WriteError(w, http.StatusMethodNotAllowed, "method_not_allowed", "method not allowed: "+r.Method) -} diff --git a/api/meta.go b/api/meta.go new file mode 100644 index 00000000..8b1a5390 --- /dev/null +++ b/api/meta.go @@ -0,0 +1,66 @@ +package api + +import ( + "net/http" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/fnamest" + storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/rule" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" +) + +type metaResponse struct { + APIVersion string `json:"api_version"` + Version string `json:"version"` + SchemaVersion int `json:"schema_version"` + ChatIDFormat string `json:"chat_id_format"` + TaskTypes []string `json:"task_types"` + TaskStatuses []string `json:"task_statuses"` + TaskItemStatuses []string `json:"task_item_statuses"` + StorageTypes []string `json:"storage_types"` + RuleTypes []string `json:"rule_types"` + FilenameStrategies []string `json:"filename_strategies"` + ConflictStrategies []string `json:"conflict_strategies"` +} + +func (s *server) meta(w http.ResponseWriter, _ *http.Request) { + taskTypes := tasktype.TaskTypeValues() + taskTypeNames := make([]string, 0, len(taskTypes)) + for _, value := range taskTypes { + taskTypeNames = append(taskTypeNames, value.String()) + } + ruleTypes := rule.Values() + ruleTypeNames := make([]string, 0, len(ruleTypes)) + for _, value := range ruleTypes { + ruleTypeNames = append(ruleTypeNames, value.String()) + } + WriteJSON(w, http.StatusOK, metaResponse{ + APIVersion: "v1", + Version: config.Version, + SchemaVersion: database.CurrentSchemaVersion, + ChatIDFormat: "bot_api", + TaskTypes: taskTypeNames, + TaskStatuses: []string{ + string(TaskStatusQueued), + string(TaskStatusRunning), + string(TaskStatusCompleted), + string(TaskStatusPartial), + string(TaskStatusFailed), + string(TaskStatusCancelled), + }, + TaskItemStatuses: []string{ + string(core.ItemStatusCompleted), + string(core.ItemStatusFailed), + string(core.ItemStatusSkipped), + string(core.ItemStatusCancelled), + }, + StorageTypes: storenum.StorageTypeNames(), + RuleTypes: ruleTypeNames, + FilenameStrategies: fnamest.FnameSTNames(), + ConflictStrategies: tcbdata.ConflictStrategyValues(), + }) +} diff --git a/api/onboarding.go b/api/onboarding.go new file mode 100644 index 00000000..71d3c281 --- /dev/null +++ b/api/onboarding.go @@ -0,0 +1,93 @@ +package api + +import ( + "net/http" + "strings" + + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/settings" + "github.com/krau/SaveAny-Bot/storage" +) + +type OnboardingSteps struct { + BotConfigured bool `json:"bot_configured"` + StorageReady bool `json:"storage_ready"` + UserReady bool `json:"user_ready"` + BotRunning bool `json:"bot_running"` + FirstTask bool `json:"first_task"` +} + +type OnboardingLegacy struct { + Imported bool `json:"imported"` + ImportedAt string `json:"imported_at,omitempty"` +} + +type OnboardingStatus struct { + Complete bool `json:"complete"` + Steps OnboardingSteps `json:"steps"` + BotState string `json:"bot_state"` + Legacy OnboardingLegacy `json:"legacy"` +} + +// onboarding reports the minimum configuration needed before the bot can +// accept a task. It intentionally exposes no credentials. +func (s *server) onboarding(w http.ResponseWriter, r *http.Request) { + cfg := s.deps.Settings.Config() + records, err := database.ListStorageRecords(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + users, err := database.GetAllUsers(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + _, taskTotal, err := database.ListTaskRecords(r.Context(), database.TaskFilter{Limit: 1}) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + + storageReady := false + initErrors := storage.LastErrors() + for _, record := range records { + if !record.Enable { + continue + } + if message, failed := initErrors[record.Name]; failed && message != "" { + continue + } + storageReady = true + break + } + userReady := false + for _, user := range users { + if user.Enabled { + userReady = true + break + } + } + + botToken := strings.TrimSpace(cfg.Telegram.Token) != "" + botStatus := s.deps.Telegram.Status(r.Context()) + importedAt, imported := settings.LegacyImportTime(r.Context()) + legacy := OnboardingLegacy{Imported: imported} + if !importedAt.IsZero() { + legacy.ImportedAt = importedAt.UTC().Format("2006-01-02T15:04:05Z07:00") + } + + status := OnboardingStatus{ + Complete: botToken && storageReady && userReady, + Steps: OnboardingSteps{ + BotConfigured: botToken, + StorageReady: storageReady, + UserReady: userReady, + BotRunning: botStatus.Bot.State == "running", + FirstTask: taskTotal > 0, + }, + BotState: botStatus.Bot.State, + Legacy: legacy, + } + WriteJSON(w, http.StatusOK, status) +} diff --git a/api/parsers.go b/api/parsers.go new file mode 100644 index 00000000..6719d2b1 --- /dev/null +++ b/api/parsers.go @@ -0,0 +1,96 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/parsers" +) + +type parserFileResponse struct { + Name string `json:"name"` + Directory string `json:"directory"` + Size int64 `json:"size"` + Modified time.Time `json:"modified"` +} + +type parserListResponse struct { + Enabled bool `json:"enabled"` + Dirs []string `json:"dirs"` + Files []parserFileResponse `json:"files"` +} + +func parserDirs() []string { + cfg := config.C() + if len(cfg.Parser.PluginDirs) > 0 { + return cfg.Parser.PluginDirs + } + return []string{"plugins"} +} + +func (s *server) listParsers(w http.ResponseWriter, r *http.Request) { + cfg := config.C() + dirs := parserDirs() + files := make([]parserFileResponse, 0) + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + continue + } + WriteError(w, http.StatusInternalServerError, "read_failed", err.Error()) + return + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".js") { + continue + } + info, err := entry.Info() + if err != nil { + continue + } + files = append(files, parserFileResponse{ + Name: entry.Name(), + Directory: dir, + Size: info.Size(), + Modified: info.ModTime(), + }) + } + } + WriteJSON(w, http.StatusOK, parserListResponse{Enabled: cfg.Parser.PluginEnable, Dirs: dirs, Files: files}) +} + +func (s *server) createParser(w http.ResponseWriter, r *http.Request) { + if !config.C().Parser.PluginEnable { + WriteError(w, http.StatusConflict, "parser_disabled", "parser plugins are disabled") + return + } + var body struct { + Filename string `json:"filename"` + Content string `json:"content"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 10<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + filename := filepath.Base(strings.TrimSpace(body.Filename)) + if filename == "" || filename == "." || !strings.HasSuffix(strings.ToLower(filename), ".js") { + WriteFieldErrors(w, http.StatusBadRequest, "filename must end with .js", map[string]string{"filename": "invalid filename"}) + return + } + if body.Content == "" { + WriteFieldErrors(w, http.StatusBadRequest, "content is required", map[string]string{"content": "required"}) + return + } + if err := parsers.AddPlugin(r.Context(), body.Content, filename); err != nil { + WriteError(w, http.StatusBadRequest, "install_failed", err.Error()) + return + } + WriteJSON(w, http.StatusCreated, map[string]any{"name": filename}) +} diff --git a/api/progress.go b/api/progress.go index f168f840..b6f55c3d 100644 --- a/api/progress.go +++ b/api/progress.go @@ -2,15 +2,18 @@ package api import ( "context" + + "github.com/krau/SaveAny-Bot/config" "sync" "time" + "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/pkg/taskevent" ) -// TaskProgressInfo stores the progress of an API-submitted task. All fields are -// guarded by mu. It implements taskevent.Sink so the task layer can update it -// without knowing about the API. +// TaskProgressInfo stores live progress for a task. All fields are guarded by +// mu. It implements taskevent.Sink so the task layer can update it without +// knowing about the API. type TaskProgressInfo struct { mu sync.Mutex TaskID string @@ -21,6 +24,7 @@ type TaskProgressInfo struct { DownloadedBytes int64 TotalFiles int DownloadedFiles int + StorageID uint Storage string Path string Error string @@ -31,8 +35,8 @@ type TaskProgressInfo struct { webhookNotified bool } -// progressStore holds all API tasks. Entries are removed a fixed duration after -// they reach a terminal state to bound memory usage. +// progressStore holds all tracked tasks. Entries are removed a fixed duration +// after they reach a terminal state to bound memory usage. type progressStore struct { mu sync.RWMutex tasks map[string]*TaskProgressInfo @@ -44,12 +48,26 @@ var store = &progressStore{ retention: 24 * time.Hour, } +// TaskProgressSink creates the in-memory observer used by tasks started +// outside the API. Their progress is rendered in the task list but no webhook +// is attached. +func TaskProgressSink(task core.Executable) taskevent.Sink { + storageName := "" + storagePath := "" + if destination, ok := task.(core.DestinationInfo); ok { + storageName = destination.StorageName() + storagePath = destination.StoragePath() + } + return RegisterTask(task.TaskID(), task.Type().String(), storageName, storagePath, task.Title(), "") +} + func RegisterTask(taskID, taskType, storage, path, title, webhook string) *TaskProgressInfo { info := &TaskProgressInfo{ TaskID: taskID, Type: taskType, Status: TaskStatusQueued, Title: title, + StorageID: config.C().GetStorageIDByName(storage), Storage: storage, Path: path, CreatedAt: time.Now(), @@ -96,7 +114,7 @@ func CleanupExpired() { defer store.mu.Unlock() for id, info := range store.tasks { info.mu.Lock() - terminal := info.Status == TaskStatusCompleted || info.Status == TaskStatusFailed || info.Status == TaskStatusCancelled + terminal := info.Status.IsTerminal() stale := terminal && now.Sub(info.UpdatedAt) > store.retention info.mu.Unlock() if stale { @@ -153,6 +171,10 @@ func (t *TaskProgressInfo) snapshot() (status TaskStatus, total, downloaded int6 // status/progress updates and fires the webhook on terminal transitions. func (t *TaskProgressInfo) Emit(e taskevent.Event) { t.mu.Lock() + if t.Status.IsTerminal() { + t.mu.Unlock() + return + } switch e.Phase { case taskevent.PhaseStart: t.Status = TaskStatusRunning @@ -175,22 +197,30 @@ func (t *TaskProgressInfo) Emit(e taskevent.Event) { t.DownloadedFiles = e.DownloadedFiles } case taskevent.PhaseDone: - if e.Err != nil { - t.Status = TaskStatusFailed - t.Error = e.Err.Error() - } else { - t.Status = TaskStatusCompleted + if t.Status != TaskStatusCancelled { + switch { + case e.Cancelled: + t.Status = TaskStatusCancelled + case e.Err != nil: + t.Status = TaskStatusFailed + t.Error = e.Err.Error() + case e.Partial: + t.Status = TaskStatusPartial + default: + t.Status = TaskStatusCompleted + } } } t.UpdatedAt = time.Now() - notify := t.Webhook != "" && !t.webhookNotified && (t.Status == TaskStatusCompleted || t.Status == TaskStatusFailed) + notify := t.Webhook != "" && !t.webhookNotified && t.Status.IsTerminal() && t.Status != TaskStatusCancelled + status := t.Status if notify { t.webhookNotified = true } t.mu.Unlock() if notify { - payload := CreateWebhookPayload(t.TaskID, t.Type, t.Status, t.Storage, t.Path, e.Err) + payload := CreateWebhookPayload(t.TaskID, t.Type, status, t.StorageID, t.Storage, t.Path, e.Err) SendWebhook(context.Background(), payload) } } diff --git a/api/router.go b/api/router.go new file mode 100644 index 00000000..4480766c --- /dev/null +++ b/api/router.go @@ -0,0 +1,148 @@ +package api + +import ( + "net/http" + "time" + + "github.com/charmbracelet/log" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" +) + +type server struct { + deps Deps + factory *TaskFactory + mediaMetadataExtractor mediaMetadataExtractor + logins *loginLimiter + startedAt time.Time +} + +// NewHandler builds the redesigned JSON API mounted under /api/v1. +func NewHandler(deps Deps) http.Handler { + s := &server{ + deps: deps, + factory: NewTaskFactory(deps.Ctx), + mediaMetadataExtractor: extractMediaMetadata, + logins: newLoginLimiter(), + startedAt: time.Now(), + } + go s.sessionCleanup(deps.Ctx) + StartCleanupLoop(deps.Ctx) + core.SetTaskSinkFactory(TaskProgressSink) + + mux := http.NewServeMux() + mux.HandleFunc("GET /health", s.health) + mux.HandleFunc("POST /api/v1/auth/login", s.login) + mux.HandleFunc("GET /api/v1/auth/token", s.tokenLogin) + mux.HandleFunc("POST /api/v1/auth/logout", s.logout) + mux.HandleFunc("GET /api/v1/auth/me", s.me) + mux.HandleFunc("POST /api/v1/auth/password", s.changePassword) + mux.HandleFunc("GET /api/v1/onboarding", s.onboarding) + mux.HandleFunc("GET /api/v1/meta", s.meta) + + mux.HandleFunc("GET /api/v1/settings", s.getSettings) + mux.HandleFunc("PUT /api/v1/settings/{group}", s.updateSettings) + + mux.HandleFunc("GET /api/v1/storages", s.listStorages) + mux.HandleFunc("POST /api/v1/storages", s.createStorage) + mux.HandleFunc("PUT /api/v1/storages/{storage_id}", s.updateStorage) + mux.HandleFunc("DELETE /api/v1/storages/{storage_id}", s.deleteStorage) + mux.HandleFunc("POST /api/v1/storages/{storage_id}/test", s.testStorage) + + mux.HandleFunc("GET /api/v1/users", s.listUsers) + mux.HandleFunc("POST /api/v1/users", s.createUser) + mux.HandleFunc("PUT /api/v1/users/{id}", s.updateUser) + mux.HandleFunc("DELETE /api/v1/users/{id}", s.deleteUser) + mux.HandleFunc("GET /api/v1/users/{id}/dirs", s.listUserDirs) + mux.HandleFunc("POST /api/v1/users/{id}/dirs", s.createUserDir) + mux.HandleFunc("DELETE /api/v1/users/{id}/dirs/{dirID}", s.deleteUserDir) + mux.HandleFunc("GET /api/v1/users/{id}/rules", s.listUserRules) + mux.HandleFunc("POST /api/v1/users/{id}/rules", s.createUserRule) + mux.HandleFunc("POST /api/v1/users/{id}/rules/presets", s.importUserRulePresets) + mux.HandleFunc("DELETE /api/v1/users/{id}/rules/{ruleID}", s.deleteUserRule) + mux.HandleFunc("GET /api/v1/users/{id}/watches", s.listUserWatches) + mux.HandleFunc("POST /api/v1/users/{id}/watches", s.createUserWatch) + mux.HandleFunc("DELETE /api/v1/users/{id}/watches/{chatID}", s.deleteUserWatch) + + mux.HandleFunc("GET /api/v1/telegram/status", s.telegramStatus) + mux.HandleFunc("GET /api/v1/telegram/workers", s.listBotWorkers) + mux.HandleFunc("POST /api/v1/telegram/workers", s.createBotWorker) + mux.HandleFunc("PUT /api/v1/telegram/workers/{id}", s.updateBotWorker) + mux.HandleFunc("DELETE /api/v1/telegram/workers/{id}", s.deleteBotWorker) + mux.HandleFunc("POST /api/v1/telegram/workers/{id}/start", s.startBotWorker) + mux.HandleFunc("POST /api/v1/telegram/workers/{id}/stop", s.stopBotWorker) + mux.HandleFunc("POST /api/v1/telegram/workers/{id}/test", s.testBotWorker) + mux.HandleFunc("POST /api/v1/telegram/bot/start", s.startBot) + mux.HandleFunc("POST /api/v1/telegram/bot/stop", s.stopBot) + mux.HandleFunc("POST /api/v1/telegram/bot/restart", s.restartBot) + mux.HandleFunc("POST /api/v1/telegram/userbot/login", s.startUserbotLogin) + mux.HandleFunc("POST /api/v1/telegram/userbot/login/{flow}/code", s.submitUserbotCode) + mux.HandleFunc("POST /api/v1/telegram/userbot/login/{flow}/password", s.submitUserbotPassword) + mux.HandleFunc("DELETE /api/v1/telegram/userbot/login/{flow}", s.cancelUserbotLogin) + mux.HandleFunc("POST /api/v1/telegram/userbot/logout", s.logoutUserbot) + mux.HandleFunc("POST /api/v1/telegram/userbot/sync-peers", s.syncUserbotPeers) + + mux.HandleFunc("GET /api/v1/tasks", s.listTasks) + mux.HandleFunc("POST /api/v1/tasks", s.createTask) + mux.HandleFunc("GET /api/v1/tasks/{id}", s.getTask) + mux.HandleFunc("GET /api/v1/tasks/{id}/metadata", s.getTaskMetadata) + mux.HandleFunc("POST /api/v1/tasks/{id}/cancel", s.cancelTask) + mux.HandleFunc("POST /api/v1/tasks/{id}/retry", s.retryTask) + mux.HandleFunc("DELETE /api/v1/tasks/{id}", s.deleteTask) + mux.HandleFunc("GET /api/v1/task-types", s.taskTypes) + mux.HandleFunc("GET /api/v1/media-metadata", s.mediaMetadata) + + mux.HandleFunc("GET /api/v1/system/info", s.systemInfo) + mux.HandleFunc("POST /api/v1/system/restart", s.restartSystem) + mux.HandleFunc("GET /api/v1/system/update", s.getUpdateInfo) + mux.HandleFunc("POST /api/v1/system/update", s.applyUpdate) + mux.HandleFunc("GET /api/v1/parsers", s.listParsers) + mux.HandleFunc("POST /api/v1/parsers", s.createParser) + mux.HandleFunc("GET /api/v1/filesystem", s.listFilesystem) + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + WriteError(w, http.StatusNotFound, "not_found", "endpoint not found: "+r.URL.Path) + }) + + var handler http.Handler = mux + handler = s.authMiddleware(handler) + handler = loggingMiddleware(handler) + handler = recoveryMiddleware(handler) + return handler +} + +func (s *server) health(w http.ResponseWriter, r *http.Request) { + WriteJSON(w, http.StatusOK, map[string]string{"status": "ok", "version": config.Version}) +} + +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + wrapped := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK} + next.ServeHTTP(wrapped, r) + log.Infof("%s %s %d %s", r.Method, r.URL.Path, wrapped.statusCode, time.Since(start)) + }) +} + +func recoveryMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := recover(); err != nil { + log.Errorf("Panic recovered: %v", err) + WriteError(w, http.StatusInternalServerError, "internal_error", "internal server error") + } + }() + next.ServeHTTP(w, r) + }) +} + +type responseWriter struct { + http.ResponseWriter + statusCode int +} + +func (rw *responseWriter) WriteHeader(code int) { + rw.statusCode = code + rw.ResponseWriter.WriteHeader(code) +} diff --git a/api/server.go b/api/server.go deleted file mode 100644 index 42c2060d..00000000 --- a/api/server.go +++ /dev/null @@ -1,160 +0,0 @@ -package api - -import ( - "context" - "fmt" - "net" - "net/http" - "time" - - "github.com/charmbracelet/log" - "github.com/krau/SaveAny-Bot/config" -) - -type Server struct { - httpServer *http.Server - factory *TaskFactory -} - -func NewServer(ctx context.Context) *Server { - cfg := config.C().API - - factory := NewTaskFactory(ctx) - handlers := NewHandlers(factory) - - mux := http.NewServeMux() - - mux.HandleFunc("/health", handlers.HealthCheckHandler) - - mux.HandleFunc("/api/v1/tasks", func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - handlers.ListTasksHandler(w, r) - case http.MethodPost: - handlers.CreateTaskHandler(w, r) - default: - MethodNotAllowedHandler(w, r) - } - }) - mux.HandleFunc("/api/v1/tasks/", func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - handlers.GetTaskHandler(w, r) - case http.MethodDelete: - handlers.CancelTaskHandler(w, r) - default: - MethodNotAllowedHandler(w, r) - } - }) - mux.HandleFunc("/api/v1/storages", handlers.ListStoragesHandler) - mux.HandleFunc("/api/v1/media-metadata", handlers.GetMediaMetadataHandler) - mux.HandleFunc("/api/v1/task-types", handlers.GetTaskTypesHandler) - - mux.HandleFunc("/", NotFoundHandler) - - var handler http.Handler = mux - - token := cfg.Token - if token != "" { - handler = AuthMiddleware()(handler) - } - - handler = loggingMiddleware(handler) - - handler = recoveryMiddleware(handler) - - return &Server{ - httpServer: &http.Server{ - Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), - Handler: handler, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 120 * time.Second, - }, - factory: factory, - } -} - -func (s *Server) Start(ctx context.Context) error { - logger := log.FromContext(ctx).With("module", "api") - - logger.Infof("Starting API server on %s", s.httpServer.Addr) - - // Bind synchronously so listen failures are returned to the caller. - ln, err := net.Listen("tcp", s.httpServer.Addr) - if err != nil { - return fmt.Errorf("failed to listen on %s: %w", s.httpServer.Addr, err) - } - - go func() { - if err := s.httpServer.Serve(ln); err != nil && err != http.ErrServerClosed { - logger.Errorf("API server error: %v", err) - } - }() - - go func() { - <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := s.httpServer.Shutdown(shutdownCtx); err != nil { - logger.Errorf("API server shutdown error: %v", err) - } - }() - - return nil -} - -func loggingMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - start := time.Now() - - wrapped := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK} - - next.ServeHTTP(wrapped, r) - - log.Infof("%s %s %d %s", r.Method, r.URL.Path, wrapped.statusCode, time.Since(start)) - }) -} - -func recoveryMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer func() { - if err := recover(); err != nil { - log.Errorf("Panic recovered: %v", err) - WriteError(w, http.StatusInternalServerError, "internal_error", "internal server error") - } - }() - next.ServeHTTP(w, r) - }) -} - -type responseWriter struct { - http.ResponseWriter - statusCode int -} - -func (rw *responseWriter) WriteHeader(code int) { - rw.statusCode = code - rw.ResponseWriter.WriteHeader(code) -} - -// Start initializes and starts the API server. It refuses to start without a -// token, since an open download proxy is a security risk. -func Start(ctx context.Context) error { - cfg := config.C().API - - if !cfg.Enable { - return nil - } - - if cfg.Token == "" { - return fmt.Errorf("API server is enabled but no token is set; refusing to start insecurely") - } - - server := NewServer(ctx) - if err := server.Start(ctx); err != nil { - return err - } - StartCleanupLoop(ctx) - return nil -} diff --git a/api/settings.go b/api/settings.go new file mode 100644 index 00000000..b3f8eb36 --- /dev/null +++ b/api/settings.go @@ -0,0 +1,39 @@ +package api + +import ( + "encoding/json" + "net/http" +) + +func (s *server) getSettings(w http.ResponseWriter, r *http.Request) { + values, err := s.deps.Settings.PublicSettings() + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusOK, values) +} + +func (s *server) updateSettings(w http.ResponseWriter, r *http.Request) { + group := r.PathValue("group") + var patch json.RawMessage + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&patch); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + restart, err := s.deps.Settings.UpdateGroup(r.Context(), group, patch) + if err != nil { + WriteError(w, http.StatusBadRequest, "settings_update_failed", err.Error()) + return + } + warnings := make([]string, 0) + if group == "telegram" { + if err := s.deps.Telegram.ApplyTelegram(r.Context()); err != nil { + warnings = append(warnings, err.Error()) + } + } + WriteJSON(w, http.StatusOK, map[string]any{ + "restart_required": restart, + "warnings": warnings, + }) +} diff --git a/api/storages.go b/api/storages.go new file mode 100644 index 00000000..3cfc78ce --- /dev/null +++ b/api/storages.go @@ -0,0 +1,142 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + "strconv" + + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/settings" + "github.com/krau/SaveAny-Bot/storage" +) + +var storageSecretKeys = map[string]struct{}{ + "password": {}, "token": {}, "secret": {}, "access_key_id": {}, "secret_access_key": {}, +} + +func maskStorageConfig(config map[string]any) map[string]any { + out := make(map[string]any, len(config)) + for key, value := range config { + if _, secret := storageSecretKeys[key]; secret { + if s, ok := value.(string); ok && s != "" { + out[key] = "__MASKED__" + continue + } + } + out[key] = value + } + return out +} + +func storageResponse(record *database.StorageRecord, initErr string) StorageRecord { + var config map[string]any + if record.ConfigJSON != "" { + if err := json.Unmarshal([]byte(record.ConfigJSON), &config); err != nil { + log.Warnf("Invalid storage config for %s: %v", record.Name, err) + } + } + if config == nil { + config = make(map[string]any) + } + return StorageRecord{ + StorageID: record.ID, + Name: record.Name, + Type: record.Type, + Enable: record.Enable, + Config: maskStorageConfig(config), + ConfigVersion: record.ConfigVersion, + Error: initErr, + } +} + +func (s *server) listStorages(w http.ResponseWriter, r *http.Request) { + records, err := database.ListStorageRecords(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + initErrors := storage.LastErrors() + out := make([]StorageRecord, 0, len(records)) + for i := range records { + out = append(out, storageResponse(&records[i], initErrors[records[i].Name])) + } + WriteJSON(w, http.StatusOK, map[string]any{"storages": out}) +} + +func (s *server) createStorage(w http.ResponseWriter, r *http.Request) { + var in StorageInput + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + record, err := s.deps.Settings.CreateStorage(r.Context(), settings.StorageInput{ + Name: in.Name, Type: in.Type, Enable: in.Enable, Config: in.Config, + }) + if err != nil { + if errors.Is(err, settings.ErrStorageExists) { + WriteError(w, http.StatusConflict, "storage_exists", err.Error()) + return + } + WriteError(w, http.StatusBadRequest, "storage_create_failed", err.Error()) + return + } + WriteJSON(w, http.StatusCreated, storageResponse(record, "")) +} + +func (s *server) updateStorage(w http.ResponseWriter, r *http.Request) { + id64, err := strconv.ParseUint(r.PathValue("storage_id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid storage id") + return + } + var in StorageInput + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + record, err := s.deps.Settings.UpdateStorage(r.Context(), uint(id64), settings.StorageInput{ + Name: in.Name, Type: in.Type, Enable: in.Enable, Config: in.Config, + }) + if err != nil { + WriteError(w, http.StatusBadRequest, "storage_update_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, storageResponse(record, "")) +} + +func (s *server) deleteStorage(w http.ResponseWriter, r *http.Request) { + id64, err := strconv.ParseUint(r.PathValue("storage_id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid storage id") + return + } + if err := s.deps.Settings.DeleteStorage(r.Context(), uint(id64)); err != nil { + if errors.Is(err, settings.ErrStorageInUse) { + WriteError(w, http.StatusConflict, "storage_in_use", err.Error()) + return + } + WriteError(w, http.StatusBadRequest, "storage_delete_failed", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) testStorage(w http.ResponseWriter, r *http.Request) { + id64, err := strconv.ParseUint(r.PathValue("storage_id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid storage id") + return + } + testCtx := r.Context() + if clientCtx, err := getClientContext(); err == nil { + testCtx = tgutil.ExtWithContext(testCtx, clientCtx) + } + if err := s.deps.Settings.TestStorage(testCtx, uint(id64)); err != nil { + WriteJSON(w, http.StatusOK, map[string]any{"ok": false, "error": err.Error()}) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"ok": true}) +} diff --git a/api/system.go b/api/system.go new file mode 100644 index 00000000..3e8f5e7b --- /dev/null +++ b/api/system.go @@ -0,0 +1,42 @@ +package api + +import ( + "net/http" + "time" + + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" +) + +func (s *server) systemInfo(w http.ResponseWriter, r *http.Request) { + cfg := s.deps.Settings.Config() + WriteJSON(w, http.StatusOK, map[string]any{ + "version": config.Version, + "commit": config.GitCommit, + "build_time": config.BuildTime, + "docker": config.Docker == "true", + "uptime_seconds": int64(time.Since(s.startedAt).Seconds()), + "data_dir": s.deps.DataDir, + "db_path": s.deps.DBPath, + "workers": cfg.Workers, + "queued_tasks": len(core.GetQueuedTasks(r.Context())), + "running_tasks": len(core.GetRunningTasks(r.Context())), + }) +} + +func (s *server) restartSystem(w http.ResponseWriter, r *http.Request) { + force, err := decodeForce(r) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if !force && core.ActiveTaskCount() > 0 { + WriteError(w, http.StatusConflict, "tasks_running", "tasks are running; use {\"force\":true} to restart anyway") + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"restarting": true}) + go func() { + time.Sleep(200 * time.Millisecond) + s.deps.Telegram.RestartProcess() + }() +} diff --git a/api/tasks.go b/api/tasks.go new file mode 100644 index 00000000..4e8039e9 --- /dev/null +++ b/api/tasks.go @@ -0,0 +1,355 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "strconv" + "time" + + "github.com/krau/SaveAny-Bot/client/bot" + userclient "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/core/retry" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/mygotg/ext" +) + +type mediaMetadataExtractor func(ctx context.Context, url string) (*MediaMetadataResponse, error) + +func (s *server) createTask(w http.ResponseWriter, r *http.Request) { + var req CreateTaskRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "failed to decode request body: "+err.Error()) + return + } + if req.Type == "" { + WriteError(w, http.StatusBadRequest, "invalid_request", "task type is required") + return + } + if req.StorageID == 0 { + WriteError(w, http.StatusBadRequest, "invalid_request", "storage_id is required") + return + } + resp, err := s.factory.CreateTask(&req) + if err != nil { + if errors.Is(err, ErrClientUnavailable) { + WriteError(w, http.StatusServiceUnavailable, "client_unavailable", err.Error()) + return + } + WriteError(w, http.StatusBadRequest, "task_creation_failed", err.Error()) + return + } + WriteJSON(w, http.StatusCreated, resp) +} + +func (s *server) listTasks(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + limit := 0 + if raw := query.Get("limit"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid limit") + return + } + limit = parsed + } + offset := 0 + if raw := query.Get("offset"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid offset") + return + } + offset = parsed + } + records, total, err := database.ListTaskRecords(r.Context(), database.TaskFilter{ + Status: query.Get("status"), + Type: query.Get("type"), + Limit: limit, + Offset: offset, + }) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + tasks := make([]TaskInfoResponse, 0, len(records)) + taskIDs := make([]string, 0, len(records)) + for i := range records { + taskIDs = append(taskIDs, records[i].TaskID) + } + failedItems, err := database.ListFailedTaskItems(r.Context(), taskIDs, 5) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + failedCounts, err := database.CountFailedTaskItems(r.Context(), taskIDs) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + metadataCounts, err := database.CountTaskMetadata(r.Context(), taskIDs) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + for i := range records { + resp := taskResponse(&records[i]) + resp.FailedItems = taskItemResponses(failedItems[records[i].TaskID], nil) + resp.FailedCount = failedCounts[records[i].TaskID] + resp.MetadataAvailable = metadataCounts[records[i].TaskID] > 0 + tasks = append(tasks, resp) + } + WriteJSON(w, http.StatusOK, TasksListResponse{Tasks: tasks, Total: total}) +} + +func (s *server) getTask(w http.ResponseWriter, r *http.Request) { + record, err := database.GetTaskRecord(r.Context(), r.PathValue("id")) + if err != nil { + WriteError(w, http.StatusNotFound, "task_not_found", "task not found") + return + } + resp := taskResponse(record) + items, err := database.ListTaskItems(r.Context(), record.TaskID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + metadataRecords, err := database.ListTaskItemMetadata(r.Context(), record.TaskID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + metadataAvailable := make(map[int]bool, len(metadataRecords)) + for _, item := range metadataRecords { + metadataAvailable[item.ItemIndex] = true + } + resp.MetadataAvailable = len(metadataRecords) > 0 + resp.Items = taskItemResponses(items, metadataAvailable) + for _, item := range items { + if item.Status == string(core.ItemStatusFailed) { + resp.FailedCount++ + } + } + WriteJSON(w, http.StatusOK, resp) +} + +func taskItemResponses(items []database.TaskItemRecord, metadataAvailable map[int]bool) []TaskItemResponse { + if len(items) == 0 { + return nil + } + out := make([]TaskItemResponse, 0, len(items)) + for _, item := range items { + out = append(out, TaskItemResponse{ + Index: item.Index, + Name: item.Name, + Path: item.Path, + StorageID: item.StorageID, + StorageName: item.StorageName, + Status: item.Status, + Error: item.Error, + Size: item.Size, + SourceChatID: item.SourceChatID, + SourceMessageID: item.SourceMsgID, + MetadataAvailable: metadataAvailable[item.Index], + }) + } + return out +} + +func (s *server) getTaskMetadata(w http.ResponseWriter, r *http.Request) { + record, err := database.GetTaskRecord(r.Context(), r.PathValue("id")) + if err != nil { + WriteError(w, http.StatusNotFound, "task_not_found", "task not found") + return + } + records, err := database.ListTaskItemMetadata(r.Context(), record.TaskID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if len(records) == 0 { + WriteError(w, http.StatusNotFound, "metadata_not_found", "no metadata recorded for this task") + return + } + taskItems, err := database.ListTaskItems(r.Context(), record.TaskID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + byIndex := make(map[int]database.TaskItemRecord, len(taskItems)) + for _, item := range taskItems { + byIndex[item.Index] = item + } + items := make([]TaskMetadataItem, 0, len(records)) + for _, item := range records { + taskItem := byIndex[item.ItemIndex] + items = append(items, TaskMetadataItem{ + Index: item.ItemIndex, + Name: taskItem.Name, + Path: taskItem.Path, + Metadata: json.RawMessage(item.JSON), + }) + } + WriteJSON(w, http.StatusOK, TaskMetadataResponse{TaskID: record.TaskID, Items: items}) +} + +func (s *server) retryTask(w http.ResponseWriter, r *http.Request) { + record, err := database.GetTaskRecord(r.Context(), r.PathValue("id")) + if err != nil { + WriteError(w, http.StatusNotFound, "task_not_found", "task not found") + return + } + newID, count, err := retry.FailedTask(s.deps.Ctx, []*ext.Context{bot.ExtContext(), userclient.GetCtx()}, record.TaskID) + switch { + case err == nil: + WriteJSON(w, http.StatusAccepted, map[string]any{"task_id": newID, "status": TaskStatusQueued, "file_count": count}) + case errors.Is(err, retry.ErrTaskActive): + WriteError(w, http.StatusConflict, "task_active", err.Error()) + case errors.Is(err, retry.ErrNothingToRetry): + WriteError(w, http.StatusConflict, "nothing_to_retry", err.Error()) + case errors.Is(err, retry.ErrUnsupportedType): + WriteError(w, http.StatusBadRequest, "retry_unsupported", err.Error()) + case errors.Is(err, retry.ErrStorageMissing): + WriteError(w, http.StatusConflict, "storage_missing", err.Error()) + case errors.Is(err, retry.ErrSourceUnavailable): + WriteError(w, http.StatusConflict, "source_unavailable", err.Error()) + case errors.Is(err, retry.ErrClientUnavailable): + WriteError(w, http.StatusServiceUnavailable, "client_unavailable", err.Error()) + default: + WriteError(w, http.StatusInternalServerError, "retry_failed", err.Error()) + } +} + +func (s *server) cancelTask(w http.ResponseWriter, r *http.Request) { + record, err := database.GetTaskRecord(r.Context(), r.PathValue("id")) + if err != nil { + WriteError(w, http.StatusNotFound, "task_not_found", "task not found") + return + } + if isTerminal(record.Status) { + WriteError(w, http.StatusConflict, "task_not_active", "task is not active") + return + } + if task, ok := GetTask(record.TaskID); ok { + task.UpdateStatus(TaskStatusCancelled) + } + if err := core.CancelTask(r.Context(), record.TaskID); err != nil { + if task, ok := GetTask(record.TaskID); ok { + task.UpdateStatus(TaskStatus(record.Status)) + } + WriteError(w, http.StatusInternalServerError, "cancel_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"task_id": record.TaskID, "status": TaskStatusCancelled}) +} + +func (s *server) deleteTask(w http.ResponseWriter, r *http.Request) { + taskID := r.PathValue("id") + record, err := database.GetTaskRecord(r.Context(), taskID) + if err != nil { + WriteError(w, http.StatusNotFound, "task_not_found", "task not found") + return + } + if !isTerminal(record.Status) { + WriteError(w, http.StatusConflict, "task_active", "cancel the task before deleting it") + return + } + if err := database.DeleteTaskRecord(r.Context(), taskID); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + DeleteTask(taskID) + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) taskTypes(w http.ResponseWriter, r *http.Request) { + types := []tasktype.TaskType{ + tasktype.TaskTypeDirectlinks, + tasktype.TaskTypeYtdlp, + tasktype.TaskTypeAria2, + tasktype.TaskTypeParseditem, + tasktype.TaskTypeTgfiles, + tasktype.TaskTypeTphpics, + tasktype.TaskTypeTransfer, + } + out := make([]map[string]string, 0, len(types)) + for _, t := range types { + out = append(out, map[string]string{"type": t.String(), "label": t.String()}) + } + WriteJSON(w, http.StatusOK, map[string]any{"types": out}) +} + +func (s *server) mediaMetadata(w http.ResponseWriter, r *http.Request) { + mediaURL := r.URL.Query().Get("url") + if mediaURL == "" { + WriteError(w, http.StatusBadRequest, "invalid_request", "url query parameter is required") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + resp, err := s.mediaMetadataExtractor(ctx, mediaURL) + if err != nil { + WriteError(w, http.StatusBadRequest, "metadata_extraction_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, resp) +} + +func isTerminal(status string) bool { + return TaskStatus(status).IsTerminal() +} + +func taskResponse(record *database.TaskRecord) TaskInfoResponse { + resp := TaskInfoResponse{ + TaskID: record.TaskID, + Type: tasktype.TaskType(record.Type), + Status: TaskStatus(record.Status), + Title: record.Title, + StorageID: record.StorageID, + StorageName: record.StorageName, + Path: record.Path, + Source: record.Source, + Error: record.Error, + CreatedAt: record.CreatedAt, + StartedAt: record.StartedAt, + FinishedAt: record.FinishedAt, + UpdatedAt: record.UpdatedAt, + } + if info, ok := GetTask(record.TaskID); ok { + status, total, downloaded, totalFiles, downloadedFiles, startedAt, errMsg, updatedAt := info.snapshot() + resp.Status = status + resp.Error = errMsg + resp.UpdatedAt = updatedAt + if resp.StartedAt == nil && !startedAt.IsZero() { + resp.StartedAt = &startedAt + } + var percent, speedMBPS float64 + if total > 0 { + percent = float64(downloaded) * 100 / float64(total) + } else if totalFiles > 0 { + percent = float64(downloadedFiles) * 100 / float64(totalFiles) + } + if !startedAt.IsZero() { + if elapsed := time.Since(startedAt).Seconds(); elapsed > 0 { + speedMBPS = float64(downloaded) / elapsed / (1024 * 1024) + } + } + if total > 0 || totalFiles > 0 { + resp.Progress = &TaskProgress{ + TotalBytes: total, + DownloadedBytes: downloaded, + TotalFiles: totalFiles, + DownloadedFiles: downloadedFiles, + Percent: percent, + SpeedMBPS: speedMBPS, + } + } + } + if resp.Progress == nil && resp.Status == TaskStatusCompleted { + resp.Progress = &TaskProgress{Percent: 100} + } + return resp +} diff --git a/api/telegram.go b/api/telegram.go new file mode 100644 index 00000000..b350ed81 --- /dev/null +++ b/api/telegram.go @@ -0,0 +1,135 @@ +package api + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" + + userclient "github.com/krau/SaveAny-Bot/client/user" +) + +func decodeForce(r *http.Request) (bool, error) { + var body struct { + Force bool `json:"force"` + } + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<20)).Decode(&body); err != nil && !errors.Is(err, io.EOF) { + return false, err + } + return body.Force, nil +} + +func (s *server) telegramStatus(w http.ResponseWriter, r *http.Request) { + WriteJSON(w, http.StatusOK, s.deps.Telegram.Status(r.Context())) +} + +func (s *server) startBot(w http.ResponseWriter, r *http.Request) { + cfg := s.deps.Settings.Config() + if strings.TrimSpace(cfg.Telegram.Token) == "" { + WriteError(w, http.StatusBadRequest, "missing_bot_token", "bot token is not configured") + return + } + if err := s.deps.Telegram.StartBot(r.Context()); err != nil { + WriteError(w, http.StatusBadRequest, "bot_start_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"state": "starting"}) +} + +func (s *server) stopBot(w http.ResponseWriter, r *http.Request) { + force, err := decodeForce(r) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if err := s.deps.Telegram.StopBot(r.Context(), force); err != nil { + WriteError(w, http.StatusConflict, "bot_stop_failed", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) restartBot(w http.ResponseWriter, r *http.Request) { + force, err := decodeForce(r) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if err := s.deps.Telegram.RestartBot(r.Context(), force); err != nil { + WriteError(w, http.StatusConflict, "bot_restart_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"state": "starting"}) +} + +func (s *server) startUserbotLogin(w http.ResponseWriter, r *http.Request) { + var body struct { + Phone string `json:"phone"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + flowID, err := s.deps.Telegram.StartUserbotLogin(r.Context(), strings.TrimSpace(body.Phone)) + if err != nil { + WriteError(w, http.StatusConflict, "userbot_login_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"flow_id": flowID}) +} + +func (s *server) submitUserbotCode(w http.ResponseWriter, r *http.Request) { + var body struct { + Code string `json:"code"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if err := s.deps.Telegram.SubmitUserbotCode(r.Context(), r.PathValue("flow"), strings.TrimSpace(body.Code)); err != nil { + WriteError(w, http.StatusConflict, "userbot_login_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"ok": true}) +} + +func (s *server) submitUserbotPassword(w http.ResponseWriter, r *http.Request) { + var body struct { + Password string `json:"password"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if err := s.deps.Telegram.SubmitUserbotPassword(r.Context(), r.PathValue("flow"), body.Password); err != nil { + WriteError(w, http.StatusConflict, "userbot_login_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{"ok": true}) +} + +func (s *server) cancelUserbotLogin(w http.ResponseWriter, r *http.Request) { + if err := s.deps.Telegram.CancelUserbotLogin(r.Context(), r.PathValue("flow")); err != nil { + WriteError(w, http.StatusConflict, "userbot_login_failed", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) logoutUserbot(w http.ResponseWriter, r *http.Request) { + if err := s.deps.Telegram.LogoutUserbot(r.Context()); err != nil { + WriteError(w, http.StatusInternalServerError, "userbot_logout_failed", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) syncUserbotPeers(w http.ResponseWriter, r *http.Request) { + count, err := userclient.SyncPeers(r.Context()) + if err != nil { + WriteError(w, http.StatusConflict, "sync_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"synced": count}) +} diff --git a/api/tgfiles.go b/api/tgfiles.go index 066a7e78..60e8b12a 100644 --- a/api/tgfiles.go +++ b/api/tgfiles.go @@ -4,16 +4,19 @@ import ( "context" "fmt" "net/url" + "regexp" "strconv" "strings" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" userclient "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/common/utils/strutil" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/ext" ) type MessageContext struct { @@ -35,10 +38,9 @@ func getClientContext() (*ext.Context, error) { } func resolveChatID(_ context.Context, idOrUsername string) (int64, error) { - if id, err := strconv.ParseInt(idOrUsername, 10, 64); err == nil { - // 私有频道 ID 需要加上 -100 前缀 - if id > 0 { - return -1000000000000 - id, nil + if id, err := strconv.ParseInt(strings.TrimPrefix(idOrUsername, "@"), 10, 64); err == nil { + if id == 0 { + return 0, fmt.Errorf("chat ID must not be zero") } return id, nil } @@ -89,7 +91,13 @@ func ParseMessageLink(ctx context.Context, link string) (int64, int, error) { if paths[0] != "c" { chatPart = paths[0] } - chatID, err := resolveChatID(ctx, chatPart) + var chatID int64 + var err error + if paths[0] == "c" { + chatID, err = tgutil.ParseChannelID(nil, chatPart) + } else { + chatID, err = resolveChatID(ctx, chatPart) + } if err != nil { return 0, 0, fmt.Errorf("failed to resolve chat ID: %w", err) } @@ -103,7 +111,7 @@ func ParseMessageLink(ctx context.Context, link string) (int64, int, error) { if paths[0] != "c" { return 0, 0, fmt.Errorf("invalid message link format: %s", link) } - chatID, err := resolveChatID(ctx, paths[1]) + chatID, err := tgutil.ParseChannelID(nil, paths[1]) if err != nil { return 0, 0, fmt.Errorf("failed to resolve chat ID: %w", err) } @@ -178,8 +186,12 @@ func ExtractFilesFromLinks(ctx context.Context, links []string) ([]tfile.TGFileM continue } - u, _ := url.Parse(link) - single := u != nil && u.Query().Has("single") + u, err := url.Parse(link) + if err != nil { + logger.Warnf("Invalid message link %s: %v", link, err) + continue + } + single := u.Query().Has("single") msgCtx, err := getMessageWithContext(ctx, chatID, msgID) if err != nil { @@ -254,3 +266,73 @@ func isValidMessageLink(link string) bool { } return false } + +// ExtractFilesFromRange resolves a chat and message ID range and returns the +// supported media messages as files. It is the web equivalent of the bot's +// batch /save syntax. +func ExtractFilesFromRange(ctx context.Context, chatRef, rangeRef, filterRef string) ([]tfile.TGFileMessage, error) { + logger := log.FromContext(ctx) + chatRef = strings.TrimSpace(chatRef) + rangeRef = strings.TrimSpace(rangeRef) + if chatRef == "" { + return nil, fmt.Errorf("chat_id is required") + } + start, end, err := strutil.ParseIntStrRange(rangeRef, "-") + if err != nil { + return nil, fmt.Errorf("invalid message range: %w", err) + } + + clientCtx, err := getClientContext() + if err != nil { + return nil, err + } + if _, err := strconv.ParseInt(strings.TrimPrefix(chatRef, "@"), 10, 64); err == nil { + if uc := userclient.GetCtx(); uc != nil { + clientCtx = uc + } + } + chatID, err := tgutil.ParseChatID(clientCtx, chatRef) + if err != nil { + return nil, fmt.Errorf("invalid chat: %w", err) + } + msgs, err := tgutil.GetMessagesRange(clientCtx, chatID, int(start), int(end)) + if err != nil { + return nil, fmt.Errorf("failed to fetch messages: %w", err) + } + if len(msgs) == 0 { + return nil, fmt.Errorf("no messages found in the specified range") + } + + var filter *regexp.Regexp + if strings.TrimSpace(filterRef) != "" { + filter, err = regexp.Compile(strings.TrimSpace(filterRef)) + if err != nil { + return nil, fmt.Errorf("invalid filter: %w", err) + } + } + files := make([]tfile.TGFileMessage, 0, len(msgs)) + for _, msg := range msgs { + if msg == nil || msg.Media == nil || !mediautil.IsSupported(msg.Media) { + continue + } + if filter != nil { + text := msg.GetMessage() + if name, err := tgutil.GetMediaFileName(msg.Media); err == nil { + text += " " + name + } + if !filter.MatchString(text) { + continue + } + } + file, err := tfile.FromMediaMessage(msg.Media, clientCtx.Raw, msg, tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*msg))) + if err != nil { + logger.Warnf("Failed to create file from message %d: %v", msg.ID, err) + continue + } + files = append(files, file) + } + if len(files) == 0 { + return nil, fmt.Errorf("no savable files found in the specified range") + } + return files, nil +} diff --git a/api/tphpics.go b/api/tphpics.go index 73a60a5b..06b0d551 100644 --- a/api/tphpics.go +++ b/api/tphpics.go @@ -2,74 +2,12 @@ package api import ( "context" - "fmt" - "net/url" - "strings" - "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/common/utils/tphutil" - "github.com/krau/SaveAny-Bot/pkg/telegraph" ) +// ExtractTelegraphImages fetches image URLs from a Telegraph page. The shared +// helper keeps the API and bot paths on the same URL validation and client. func ExtractTelegraphImages(ctx context.Context, pageURL string) ([]string, string, error) { - logger := log.FromContext(ctx) - - if !isValidTelegraphURL(pageURL) { - return nil, "", fmt.Errorf("invalid telegraph URL format: %s", pageURL) - } - - pagepath, err := parseTelegraphPath(pageURL) - if err != nil { - return nil, "", err - } - - logger.Debugf("Fetching telegraph page: %s", pagepath) - - client := telegraph.NewClient() - page, err := client.GetPage(ctx, pagepath) - if err != nil { - return nil, "", fmt.Errorf("failed to get telegraph page: %w", err) - } - - var imgs []string - for _, elem := range page.Content { - imgs = append(imgs, tphutil.GetNodeImages(elem)...) - } - - if len(imgs) == 0 { - return nil, "", fmt.Errorf("no images found in telegraph page") - } - - return imgs, pagepath, nil -} - -func parseTelegraphPath(pageURL string) (string, error) { - u, err := url.Parse(pageURL) - if err != nil { - return "", fmt.Errorf("invalid telegraph URL: %w", err) - } - - if !strings.HasSuffix(u.Host, "telegra.ph") && !strings.HasSuffix(u.Host, "telegraph.co") { - return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host) - } - - paths := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") - if len(paths) == 0 || paths[0] == "" { - return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path) - } - - pagepath := paths[len(paths)-1] - pagepath, err = url.PathUnescape(pagepath) - if err != nil { - return "", fmt.Errorf("failed to unescape telegraph path: %w", err) - } - - return strings.TrimSpace(pagepath), nil -} - -func isValidTelegraphURL(url string) bool { - return strings.HasPrefix(url, "https://telegra.ph/") || - strings.HasPrefix(url, "http://telegra.ph/") || - strings.HasPrefix(url, "https://telegraph.co/") || - strings.HasPrefix(url, "http://telegraph.co/") + return tphutil.ExtractImages(ctx, pageURL) } diff --git a/api/types.go b/api/types.go index 7b82909d..e7b70b3e 100644 --- a/api/types.go +++ b/api/types.go @@ -1,36 +1,112 @@ package api import ( + "context" "encoding/json" "net/http" "time" + "github.com/krau/SaveAny-Bot/pkg/enums/storagefilter" + "github.com/krau/SaveAny-Bot/pkg/enums/taskstatus" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/settings" ) -type TaskStatus string +type Deps struct { + Ctx context.Context + Settings *settings.Service + Telegram TelegramController + DataDir string + DBPath string + WebToken string +} + +type StorageInput struct { + Name string `json:"name"` + Type string `json:"type"` + Enable bool `json:"enable"` + Config map[string]any `json:"config"` +} + +type StorageRecord struct { + StorageID uint `json:"storage_id"` + Name string `json:"name"` + Type string `json:"type"` + Enable bool `json:"enable"` + Config map[string]any `json:"config"` + ConfigVersion int `json:"config_version,omitempty"` + Error string `json:"error,omitempty"` +} + +type BotStatus struct { + State string `json:"state"` + ID int64 `json:"id,omitempty"` + Username string `json:"username,omitempty"` + Error string `json:"error,omitempty"` + StartedAt time.Time `json:"started_at"` +} + +type LoginFlow struct { + FlowID string `json:"flow_id"` + State string `json:"state"` + Error string `json:"error,omitempty"` +} + +type UserbotStatus struct { + Enabled bool `json:"enabled"` + State string `json:"state"` + ID int64 `json:"id,omitempty"` + Username string `json:"username,omitempty"` + Phone string `json:"phone,omitempty"` + Error string `json:"error,omitempty"` + Login *LoginFlow `json:"login,omitempty"` +} + +type TelegramStatus struct { + Bot BotStatus `json:"bot"` + Userbot UserbotStatus `json:"userbot"` +} + +type TelegramController interface { + Status(ctx context.Context) TelegramStatus + StartBot(ctx context.Context) error + StopBot(ctx context.Context, force bool) error + RestartBot(ctx context.Context, force bool) error + StartUserbotLogin(ctx context.Context, phone string) (flowID string, err error) + SubmitUserbotCode(ctx context.Context, flowID, code string) error + SubmitUserbotPassword(ctx context.Context, flowID, password string) error + CancelUserbotLogin(ctx context.Context, flowID string) error + LogoutUserbot(ctx context.Context) error + ApplyTelegram(ctx context.Context) error + RestartProcess() +} + +type TaskStatus = taskstatus.Status const ( - TaskStatusQueued TaskStatus = "queued" - TaskStatusRunning TaskStatus = "running" - TaskStatusCompleted TaskStatus = "completed" - TaskStatusFailed TaskStatus = "failed" - TaskStatusCancelled TaskStatus = "cancelled" + TaskStatusQueued = taskstatus.Queued + TaskStatusRunning = taskstatus.Running + TaskStatusCompleted = taskstatus.Completed + TaskStatusPartial = taskstatus.Partial + TaskStatusFailed = taskstatus.Failed + TaskStatusCancelled = taskstatus.Cancelled ) type CreateTaskRequest struct { - Type tasktype.TaskType `json:"type"` - Storage string `json:"storage"` - Path string `json:"path"` - Webhook string `json:"webhook,omitempty"` - Params json.RawMessage `json:"params"` + Type tasktype.TaskType `json:"type"` + StorageID uint `json:"storage_id"` + Path string `json:"path"` + Webhook string `json:"webhook,omitempty"` + Params json.RawMessage `json:"params"` } type CreateTaskResponse struct { - TaskID string `json:"task_id"` - Type tasktype.TaskType `json:"type"` - Status TaskStatus `json:"status"` - CreatedAt time.Time `json:"created_at"` + TaskID string `json:"task_id"` + Type tasktype.TaskType `json:"type"` + Status TaskStatus `json:"status"` + StorageID uint `json:"storage_id"` + StorageName string `json:"storage_name"` + CreatedAt time.Time `json:"created_at"` } type TaskProgress struct { @@ -43,45 +119,107 @@ type TaskProgress struct { } type TaskInfoResponse struct { - TaskID string `json:"task_id"` - Type tasktype.TaskType `json:"type"` - Status TaskStatus `json:"status"` - Title string `json:"title"` - Progress *TaskProgress `json:"progress,omitempty"` - Storage string `json:"storage"` - Path string `json:"path"` - Error string `json:"error,omitempty"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` + TaskID string `json:"task_id"` + Type tasktype.TaskType `json:"type"` + Status TaskStatus `json:"status"` + Title string `json:"title"` + Progress *TaskProgress `json:"progress,omitempty"` + StorageID uint `json:"storage_id"` + StorageName string `json:"storage_name"` + Path string `json:"path"` + Source string `json:"source,omitempty"` + Error string `json:"error,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + Items []TaskItemResponse `json:"items,omitempty"` + FailedItems []TaskItemResponse `json:"failed_items,omitempty"` + FailedCount int `json:"failed_count,omitempty"` + MetadataAvailable bool `json:"metadata_available,omitempty"` +} + +type TaskItemResponse struct { + Index int `json:"index"` + Name string `json:"name"` + Path string `json:"path,omitempty"` + StorageID uint `json:"storage_id,omitempty"` + StorageName string `json:"storage_name,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Size int64 `json:"size,omitempty"` + SourceChatID int64 `json:"source_chat_id,omitempty"` + SourceMessageID int `json:"source_message_id,omitempty"` + MetadataAvailable bool `json:"metadata_available,omitempty"` +} + +type TaskMetadataResponse struct { + TaskID string `json:"task_id"` + Items []TaskMetadataItem `json:"items"` +} + +type TaskMetadataItem struct { + Index int `json:"index"` + Name string `json:"name,omitempty"` + Path string `json:"path,omitempty"` + Metadata json.RawMessage `json:"metadata"` } type TasksListResponse struct { Tasks []TaskInfoResponse `json:"tasks"` - Total int `json:"total"` + Total int64 `json:"total"` +} + +type UserStorageFilter struct { + StorageIDs []uint `json:"storage_ids"` + StorageNames []string `json:"storage_names,omitempty"` + Mode storagefilter.Mode `json:"mode"` } -type StoragesResponse struct { - Storages []StorageInfo `json:"storages"` +type UserResponse struct { + ID uint `json:"id"` + ChatID int64 `json:"chat_id"` + Enabled bool `json:"enabled"` + Silent bool `json:"silent"` + DefaultStorageID uint `json:"default_storage_id"` + DefaultStorageName string `json:"default_storage_name,omitempty"` + DefaultDirID uint `json:"default_dir_id"` + ApplyRule bool `json:"apply_rule"` + FilenameStrategy string `json:"filename_strategy"` + FilenameTemplate string `json:"filename_template"` + ConflictStrategy string `json:"conflict_strategy"` + WatchNotify bool `json:"watch_notify"` + StorageFilter UserStorageFilter `json:"storage_filter"` } -type StorageInfo struct { - Name string `json:"name"` - Type string `json:"type"` +type UserPatch struct { + Enabled *bool `json:"enabled,omitempty"` + Silent *bool `json:"silent,omitempty"` + DefaultStorageID *uint `json:"default_storage_id,omitempty"` + DefaultDirID *uint `json:"default_dir_id,omitempty"` + ApplyRule *bool `json:"apply_rule,omitempty"` + FilenameStrategy *string `json:"filename_strategy,omitempty"` + FilenameTemplate *string `json:"filename_template,omitempty"` + ConflictStrategy *string `json:"conflict_strategy,omitempty"` + WatchNotify *bool `json:"watch_notify,omitempty"` + StorageFilter *UserStorageFilter `json:"storage_filter,omitempty"` } type WebhookPayload struct { TaskID string `json:"task_id"` Type string `json:"type"` Status TaskStatus `json:"status"` - Storage string `json:"storage"` + StorageID uint `json:"storage_id"` + StorageName string `json:"storage_name"` Path string `json:"path"` CompletedAt *time.Time `json:"completed_at,omitempty"` Error string `json:"error,omitempty"` } type ErrorResponse struct { - Error string `json:"error"` - Message string `json:"message,omitempty"` + Error string `json:"error"` + Message string `json:"message,omitempty"` + Fields map[string]string `json:"fields,omitempty"` } type APIError struct { @@ -90,9 +228,7 @@ type APIError struct { Message string } -func (e *APIError) Error() string { - return e.Message -} +func (e *APIError) Error() string { return e.Message } func WriteJSON(w http.ResponseWriter, statusCode int, data any) error { w.Header().Set("Content-Type", "application/json") @@ -101,10 +237,11 @@ func WriteJSON(w http.ResponseWriter, statusCode int, data any) error { } func WriteError(w http.ResponseWriter, statusCode int, errCode, message string) error { - return WriteJSON(w, statusCode, ErrorResponse{ - Error: errCode, - Message: message, - }) + return WriteJSON(w, statusCode, ErrorResponse{Error: errCode, Message: message}) +} + +func WriteFieldErrors(w http.ResponseWriter, statusCode int, message string, fields map[string]string) error { + return WriteJSON(w, statusCode, ErrorResponse{Error: "validation_failed", Message: message, Fields: fields}) } type DirectLinksParams struct { @@ -126,14 +263,17 @@ type ParsedParams struct { } type TransferParams struct { - SourceStorage string `json:"source_storage"` - SourcePath string `json:"source_path"` - TargetStorage string `json:"target_storage"` - TargetPath string `json:"target_path"` + SourceStorageID uint `json:"source_storage_id"` + SourcePath string `json:"source_path"` + TargetStorageID uint `json:"target_storage_id"` + TargetPath string `json:"target_path"` } type TGFilesParams struct { MessageLinks []string `json:"message_links"` + ChatID string `json:"chat_id,omitempty"` + MessageRange string `json:"message_range,omitempty"` + Filter string `json:"filter,omitempty"` } type TPHPicsParams struct { diff --git a/api/update.go b/api/update.go new file mode 100644 index 00000000..3696c5ae --- /dev/null +++ b/api/update.go @@ -0,0 +1,135 @@ +package api + +import ( + "errors" + "net/http" + "sync" + "time" + + "github.com/blang/semver" + "github.com/krau/SaveAny-Bot/common/selfupdate" + "github.com/krau/SaveAny-Bot/config" + "github.com/unvgo/ghselfupdate" +) + +var updateMu sync.Mutex + +type updateInfoResponse struct { + Current string `json:"current"` + Latest string `json:"latest,omitempty"` + Available bool `json:"available"` + Updatable bool `json:"updatable"` + Docker bool `json:"docker"` + ReleaseNote string `json:"release_notes,omitempty"` + URL string `json:"url,omitempty"` + PublishedAt *time.Time `json:"published_at,omitempty"` + Size int64 `json:"size,omitempty"` + Reason string `json:"reason,omitempty"` + ReasonCode string `json:"reason_code,omitempty"` +} + +func checkUpdate() (*updateInfoResponse, error) { + info := &updateInfoResponse{ + Current: config.Version, + Docker: config.Docker == "true", + } + current, err := semver.Parse(config.Version) + if err != nil { + info.Reason = err.Error() + info.ReasonCode = "invalid_version" + return info, nil + } + info.Current = current.String() + if err := selfupdate.Updatable(); err != nil { + info.Reason = err.Error() + switch { + case errors.Is(err, selfupdate.ErrDocker): + info.ReasonCode = "docker" + case errors.Is(err, selfupdate.ErrDevBuild): + info.ReasonCode = "dev_build" + case errors.Is(err, selfupdate.ErrNotWritable): + info.ReasonCode = "not_writable" + default: + info.ReasonCode = "not_updatable" + } + return info, nil + } + info.Updatable = true + latest, ok, err := ghselfupdate.DetectLatest(config.GitRepo) + if err != nil { + return nil, err + } + if !ok { + info.ReasonCode = "no_release" + info.Reason = "no release found" + return info, nil + } + info.Latest = latest.Version.String() + info.ReleaseNote = latest.ReleaseNotes + info.URL = latest.AssetURL + info.Size = int64(latest.AssetByteSize) + info.PublishedAt = latest.PublishedAt + info.Available = latest.Version.Major == current.Major && latest.Version.GT(current) + if latest.Version.Major > current.Major { + info.Reason = "major upgrade required" + info.ReasonCode = "major_upgrade_required" + } else if !latest.Version.GT(current) { + info.Reason = "already on the latest version" + info.ReasonCode = "already_latest" + } + return info, nil +} + +func (s *server) getUpdateInfo(w http.ResponseWriter, r *http.Request) { + info, err := checkUpdate() + if err != nil { + WriteError(w, http.StatusBadGateway, "update_check_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, info) +} + +func (s *server) applyUpdate(w http.ResponseWriter, r *http.Request) { + updateMu.Lock() + defer updateMu.Unlock() + + current, err := semver.Parse(config.Version) + if err != nil { + WriteError(w, http.StatusConflict, "not_updatable", err.Error()) + return + } + if err := selfupdate.Updatable(); err != nil { + WriteError(w, http.StatusConflict, "not_updatable", err.Error()) + return + } + latest, ok, err := ghselfupdate.DetectLatest(config.GitRepo) + if err != nil { + WriteError(w, http.StatusBadGateway, "update_check_failed", err.Error()) + return + } + if !ok { + WriteError(w, http.StatusNotFound, "no_release", "no release found") + return + } + if latest.Version.Major > current.Major { + WriteError(w, http.StatusConflict, "major_upgrade_required", "major upgrade required") + return + } + if !latest.Version.GT(current) { + WriteError(w, http.StatusConflict, "already_latest", "already on latest version") + return + } + release, err := selfupdate.Update(current) + if err != nil { + WriteError(w, http.StatusInternalServerError, "update_failed", err.Error()) + return + } + WriteJSON(w, http.StatusAccepted, map[string]any{ + "version": release.Version.String(), + "restarting": true, + }) + go func() { + time.Sleep(200 * time.Millisecond) + s.deps.Telegram.RestartProcess() + }() +} diff --git a/api/user_automation.go b/api/user_automation.go new file mode 100644 index 00000000..c00336fd --- /dev/null +++ b/api/user_automation.go @@ -0,0 +1,442 @@ +package api + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + "strconv" + "strings" + + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/rule" +) + +type userDirResponse struct { + ID uint `json:"id"` + StorageID uint `json:"storage_id"` + StorageName string `json:"storage_name,omitempty"` + Path string `json:"path"` +} + +type userRuleResponse struct { + ID uint `json:"id"` + Type string `json:"type"` + Content string `json:"content"` + StorageID uint `json:"storage_id"` + StorageName string `json:"storage_name,omitempty"` + DirPath string `json:"dir_path"` +} + +type userWatchResponse struct { + ID uint `json:"id"` + ChatID int64 `json:"chat_id"` + MessageFilter string `json:"message_filter,omitempty"` +} + +func (s *server) requireUser(w http.ResponseWriter, r *http.Request) (*database.User, bool) { + id, err := strconv.ParseUint(r.PathValue("id"), 10, 32) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid user id") + return nil, false + } + user, err := database.GetUserByID(r.Context(), uint(id)) + if err != nil { + WriteError(w, http.StatusNotFound, "user_not_found", "user not found") + return nil, false + } + return user, true +} + +func userDirsResponse(dirs []database.Dir, namesByID map[uint]string) []userDirResponse { + out := make([]userDirResponse, 0, len(dirs)) + for _, dir := range dirs { + out = append(out, userDirResponse{ + ID: dir.ID, + StorageID: dir.StorageID, + StorageName: namesByID[dir.StorageID], + Path: dir.Path, + }) + } + return out +} + +func (s *server) listUserDirs(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + dirs, err := database.GetUserDirs(r.Context(), user.ID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"dirs": userDirsResponse(dirs, names)}) +} + +func (s *server) createUserDir(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + var body struct { + StorageID uint `json:"storage_id"` + Path string `json:"path"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + body.Path = strings.TrimSpace(body.Path) + if body.StorageID == 0 || body.Path == "" { + WriteFieldErrors(w, http.StatusBadRequest, "storage_id and path are required", map[string]string{ + "storage_id": "required", + "path": "required", + }) + return + } + if _, err := database.GetStorageRecord(r.Context(), body.StorageID); err != nil { + WriteFieldErrors(w, http.StatusBadRequest, "unknown storage", map[string]string{"storage_id": "unknown storage"}) + return + } + existing, err := database.GetDirsByUserIDAndStorageID(r.Context(), user.ID, body.StorageID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + for _, dir := range existing { + if dir.Path == body.Path { + WriteError(w, http.StatusConflict, "dir_exists", "directory already exists") + return + } + } + if err := database.CreateDirForUser(r.Context(), user.ID, body.StorageID, body.Path); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + dirs, err := database.GetUserDirs(r.Context(), user.ID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + for _, dir := range dirs { + if dir.StorageID == body.StorageID && dir.Path == body.Path { + WriteJSON(w, http.StatusCreated, userDirResponse{ + ID: dir.ID, + StorageID: dir.StorageID, + StorageName: names[dir.StorageID], + Path: dir.Path, + }) + return + } + } + WriteError(w, http.StatusInternalServerError, "internal_error", "directory was not persisted") +} + +func (s *server) deleteUserDir(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + dirID, err := strconv.ParseUint(r.PathValue("dirID"), 10, 32) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid directory id") + return + } + if err := database.DeleteDirByID(r.Context(), uint(dirID), user.ID); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if user.DefaultDirID == uint(dirID) { + user.DefaultDirID = 0 + if err := database.UpdateUser(r.Context(), user); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + } + w.WriteHeader(http.StatusNoContent) +} + +func userRulesResponse(rules []database.Rule, namesByID map[uint]string) []userRuleResponse { + out := make([]userRuleResponse, 0, len(rules)) + for _, rule := range rules { + out = append(out, userRuleResponse{ + ID: rule.ID, + Type: rule.Type, + Content: rule.Content, + StorageID: rule.StorageID, + StorageName: namesByID[rule.StorageID], + DirPath: rule.DirPath, + }) + } + return out +} + +func validateRuleContent(ruleType, content string) error { + switch ruleType { + case rule.FileNameRegex.String(), rule.MessageRegex.String(): + if _, err := regexp.Compile(content); err != nil { + return fmt.Errorf("invalid regular expression: %w", err) + } + case rule.IsAlbum.String(): + if _, err := strconv.ParseBool(content); err != nil { + return errors.New("IS-ALBUM content must be true or false") + } + default: + return fmt.Errorf("unsupported rule type: %s", ruleType) + } + return nil +} + +func normalizeRuleType(raw string) (string, error) { + for _, typ := range rule.Values() { + if strings.EqualFold(strings.TrimSpace(raw), typ.String()) { + return typ.String(), nil + } + } + return "", fmt.Errorf("unsupported rule type: %s", raw) +} + +func (s *server) listUserRules(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + rules, err := database.GetRulesByUserID(r.Context(), user.ID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"rules": userRulesResponse(rules, names)}) +} + +func (s *server) createUserRule(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + var body struct { + Type string `json:"type"` + Content string `json:"content"` + StorageID uint `json:"storage_id"` + DirPath string `json:"dir_path"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + ruleType, err := normalizeRuleType(body.Type) + if err != nil { + WriteFieldErrors(w, http.StatusBadRequest, err.Error(), map[string]string{"type": err.Error()}) + return + } + body.Content = strings.TrimSpace(body.Content) + if err := validateRuleContent(ruleType, body.Content); err != nil { + WriteFieldErrors(w, http.StatusBadRequest, err.Error(), map[string]string{"content": err.Error()}) + return + } + if body.StorageID != 0 { + if _, err := database.GetStorageRecord(r.Context(), body.StorageID); err != nil { + WriteFieldErrors(w, http.StatusBadRequest, "unknown storage", map[string]string{"storage_id": "unknown storage"}) + return + } + } + record := &database.Rule{ + UserID: user.ID, + Type: ruleType, + Content: body.Content, + StorageID: body.StorageID, + DirPath: strings.TrimSpace(body.DirPath), + } + if err := database.CreateRule(r.Context(), record); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusCreated, userRuleResponse{ + ID: record.ID, Type: record.Type, Content: record.Content, + StorageID: record.StorageID, StorageName: names[record.StorageID], + DirPath: record.DirPath, + }) +} + +func (s *server) deleteUserRule(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + ruleID, err := strconv.ParseUint(r.PathValue("ruleID"), 10, 32) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid rule id") + return + } + if err := database.DeleteRule(r.Context(), uint(ruleID), user.ID); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) importUserRulePresets(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + var body struct { + StorageID uint `json:"storage_id"` + BasePath string `json:"base_path"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if body.StorageID != 0 { + if _, err := database.GetStorageRecord(r.Context(), body.StorageID); err != nil { + WriteFieldErrors(w, http.StatusBadRequest, "unknown storage", map[string]string{"storage_id": "unknown storage"}) + return + } + } + presets := rule.PresetCategories(strings.TrimSpace(body.BasePath)) + created := 0 + for _, preset := range presets { + record := &database.Rule{ + UserID: user.ID, + Type: rule.FileNameRegex.String(), + Content: preset.Regex, + StorageID: body.StorageID, + DirPath: preset.Dir, + } + if err := database.CreateRule(r.Context(), record); err != nil { + continue + } + created++ + } + if created == 0 { + WriteError(w, http.StatusInternalServerError, "internal_error", "failed to create preset rules") + return + } + WriteJSON(w, http.StatusCreated, map[string]any{"created": created}) +} + +func userWatchesResponse(chats []database.WatchChat) []userWatchResponse { + out := make([]userWatchResponse, 0, len(chats)) + for _, chat := range chats { + out = append(out, userWatchResponse{ID: chat.ID, ChatID: chat.ChatID, MessageFilter: chat.MessageFilter}) + } + return out +} + +func normalizeWatchFilter(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", nil + } + filterType, filterData, ok := strings.Cut(raw, ":") + if !ok || !strings.EqualFold(filterType, "msgre") || filterData == "" { + return "", errors.New("filter must be msgre:") + } + if _, err := regexp.Compile(filterData); err != nil { + return "", fmt.Errorf("invalid filter expression: %w", err) + } + return "msgre:" + filterData, nil +} + +func (s *server) listUserWatches(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + WriteJSON(w, http.StatusOK, map[string]any{"watches": userWatchesResponse(user.WatchChats)}) +} + +func (s *server) createUserWatch(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + var body struct { + ChatID int64 `json:"chat_id"` + Chat string `json:"chat"` + Filter string `json:"filter"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if body.ChatID == 0 && strings.TrimSpace(body.Chat) != "" { + chatID, err := resolveChatID(r.Context(), strings.TrimSpace(body.Chat)) + if err != nil { + WriteFieldErrors(w, http.StatusBadRequest, "invalid chat", map[string]string{"chat": err.Error()}) + return + } + body.ChatID = chatID + } + if body.ChatID == 0 { + WriteFieldErrors(w, http.StatusBadRequest, "chat is required", map[string]string{"chat": "required"}) + return + } + filter, err := normalizeWatchFilter(body.Filter) + if err != nil { + WriteFieldErrors(w, http.StatusBadRequest, err.Error(), map[string]string{"filter": err.Error()}) + return + } + watching, err := user.WatchingChat(r.Context(), body.ChatID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if watching { + WriteError(w, http.StatusConflict, "watch_exists", "chat is already watched") + return + } + if err := user.WatchChat(r.Context(), database.WatchChat{ + UserID: user.ID, + ChatID: body.ChatID, + MessageFilter: filter, + }); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + refreshed, err := database.GetUserByID(r.Context(), user.ID) + if err == nil { + user = refreshed + } + WriteJSON(w, http.StatusCreated, userWatchesResponse(user.WatchChats)) +} + +func (s *server) deleteUserWatch(w http.ResponseWriter, r *http.Request) { + user, ok := s.requireUser(w, r) + if !ok { + return + } + chatID, err := strconv.ParseInt(r.PathValue("chatID"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid chat id") + return + } + if err := user.UnwatchChat(r.Context(), chatID); err != nil { + WriteError(w, http.StatusNotFound, "watch_not_found", "watched chat not found") + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/api/users.go b/api/users.go new file mode 100644 index 00000000..314ff743 --- /dev/null +++ b/api/users.go @@ -0,0 +1,248 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/storagefilter" +) + +func (s *server) listUsers(w http.ResponseWriter, r *http.Request) { + users, err := database.GetAllUsers(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + out := make([]UserResponse, 0, len(users)) + for i := range users { + out = append(out, userResponse(&users[i], names)) + } + WriteJSON(w, http.StatusOK, map[string]any{"users": out}) +} + +func (s *server) createUser(w http.ResponseWriter, r *http.Request) { + var body struct { + ChatID int64 `json:"chat_id"` + UserPatch + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if body.ChatID <= 0 { + WriteFieldErrors(w, http.StatusBadRequest, "chat_id must be a positive Telegram user ID", map[string]string{"chat_id": "invalid"}) + return + } + if _, err := database.GetUserByChatID(r.Context(), body.ChatID); err == nil { + WriteError(w, http.StatusConflict, "user_exists", "user already exists") + return + } else if !database.IsNotFound(err) { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + candidate := &database.User{ChatID: body.ChatID, Enabled: true} + if fields := applyUserPatch(r.Context(), candidate, body.UserPatch); len(fields) > 0 { + WriteFieldErrors(w, http.StatusBadRequest, "invalid user fields", fields) + return + } + if err := database.CreateUser(r.Context(), body.ChatID); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + user, err := database.GetUserByChatID(r.Context(), body.ChatID) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if fields := applyUserPatch(r.Context(), user, body.UserPatch); len(fields) > 0 { + WriteFieldErrors(w, http.StatusBadRequest, "invalid user fields", fields) + return + } + if err := database.UpdateUser(r.Context(), user); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if err := s.deps.Settings.ReloadConfig(r.Context()); err != nil { + log.FromContext(r.Context()).Warnf("Failed to reload settings after user creation: %v", err) + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusCreated, userResponse(user, names)) +} + +func (s *server) updateUser(w http.ResponseWriter, r *http.Request) { + id64, err := strconv.ParseUint(r.PathValue("id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid user id") + return + } + user, err := database.GetUserByID(r.Context(), uint(id64)) + if err != nil { + WriteError(w, http.StatusNotFound, "user_not_found", "user not found") + return + } + var body UserPatch + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&body); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + if fields := applyUserPatch(r.Context(), user, body); len(fields) > 0 { + WriteFieldErrors(w, http.StatusBadRequest, "invalid user fields", fields) + return + } + if err := database.UpdateUser(r.Context(), user); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if err := s.deps.Settings.ReloadConfig(r.Context()); err != nil { + log.FromContext(r.Context()).Warnf("Failed to reload settings after user update: %v", err) + } + names, err := storageNamesByID(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + WriteJSON(w, http.StatusOK, userResponse(user, names)) +} + +func (s *server) deleteUser(w http.ResponseWriter, r *http.Request) { + id64, err := strconv.ParseUint(r.PathValue("id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid user id") + return + } + user, err := database.GetUserByID(r.Context(), uint(id64)) + if err != nil { + WriteError(w, http.StatusNotFound, "user_not_found", "user not found") + return + } + if err := database.DeleteUser(r.Context(), user); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if err := s.deps.Settings.ReloadConfig(r.Context()); err != nil { + log.FromContext(r.Context()).Warnf("Failed to reload settings after user deletion: %v", err) + } + w.WriteHeader(http.StatusNoContent) +} + +// storageNamesByID resolves names for both enabled and disabled storage +// records so references remain readable after a storage is disabled. +func storageNamesByID(ctx context.Context) (map[uint]string, error) { + records, err := database.ListStorageRecords(ctx) + if err != nil { + return nil, err + } + names := make(map[uint]string, len(records)) + for _, record := range records { + names[record.ID] = record.Name + } + return names, nil +} + +func userResponse(user *database.User, namesByID map[uint]string) UserResponse { + ids := make([]uint, 0) + if user.StorageFilterIDsJSON != "" { + if err := json.Unmarshal([]byte(user.StorageFilterIDsJSON), &ids); err != nil { + log.Warnf("Invalid storage filter for user %d: %v", user.ChatID, err) + } + } + if ids == nil { + ids = make([]uint, 0) + } + names := make([]string, 0, len(ids)) + for _, id := range ids { + names = append(names, namesByID[id]) + } + return UserResponse{ + ID: user.ID, + ChatID: user.ChatID, + Enabled: user.Enabled, + Silent: user.Silent, + DefaultStorageID: user.DefaultStorageID, + DefaultStorageName: namesByID[user.DefaultStorageID], + DefaultDirID: user.DefaultDirID, + ApplyRule: user.ApplyRule, + FilenameStrategy: user.FilenameStrategy, + FilenameTemplate: user.FilenameTemplate, + ConflictStrategy: user.ConflictStrategy, + WatchNotify: user.WatchNotify, + StorageFilter: UserStorageFilter{StorageIDs: ids, StorageNames: names, Mode: user.StorageFilterMode.Normalize()}, + } +} + +func applyUserPatch(ctx context.Context, user *database.User, patch UserPatch) map[string]string { + fields := make(map[string]string) + if patch.Enabled != nil { + user.Enabled = *patch.Enabled + } + if patch.Silent != nil { + user.Silent = *patch.Silent + } + if patch.DefaultStorageID != nil { + if *patch.DefaultStorageID != 0 { + if _, err := database.GetStorageRecord(ctx, *patch.DefaultStorageID); err != nil { + fields["default_storage_id"] = "unknown storage" + } + } + if _, exists := fields["default_storage_id"]; !exists { + user.DefaultStorageID = *patch.DefaultStorageID + } + } + if patch.DefaultDirID != nil { + user.DefaultDirID = *patch.DefaultDirID + } + if patch.ApplyRule != nil { + user.ApplyRule = *patch.ApplyRule + } + if patch.FilenameStrategy != nil { + user.FilenameStrategy = *patch.FilenameStrategy + } + if patch.FilenameTemplate != nil { + user.FilenameTemplate = *patch.FilenameTemplate + } + if patch.ConflictStrategy != nil { + user.ConflictStrategy = *patch.ConflictStrategy + } + if patch.WatchNotify != nil { + user.WatchNotify = *patch.WatchNotify + } + if patch.StorageFilter != nil { + for _, id := range patch.StorageFilter.StorageIDs { + if _, err := database.GetStorageRecord(ctx, id); err != nil { + fields["storage_filter.storage_ids"] = "unknown storage" + break + } + } + mode := patch.StorageFilter.Mode + if mode == "" { + mode = storagefilter.Blacklist + } else if !mode.Valid() { + fields["storage_filter.mode"] = "must be blacklist or whitelist" + } + storageIDs := patch.StorageFilter.StorageIDs + if storageIDs == nil { + storageIDs = make([]uint, 0) + } + raw, err := json.Marshal(storageIDs) + if err != nil { + fields["storage_filter.storage_ids"] = err.Error() + } else { + user.StorageFilterIDsJSON = string(raw) + user.StorageFilterMode = mode + } + } + return fields +} diff --git a/api/webhook.go b/api/webhook.go index fed12f91..0f90df28 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -87,16 +87,17 @@ func SendWebhook(ctx context.Context, payload *WebhookPayload) { }() } -func CreateWebhookPayload(taskID string, taskType string, status TaskStatus, storage, path string, err error) *WebhookPayload { +func CreateWebhookPayload(taskID string, taskType string, status TaskStatus, storageID uint, storage, path string, err error) *WebhookPayload { payload := &WebhookPayload{ - TaskID: taskID, - Type: taskType, - Status: status, - Storage: storage, - Path: path, + TaskID: taskID, + Type: taskType, + Status: status, + StorageID: storageID, + StorageName: storage, + Path: path, } - if status == TaskStatusCompleted || status == TaskStatusFailed { + if status.IsTerminal() && status != TaskStatusCancelled { now := time.Now() payload.CompletedAt = &now } diff --git a/api/workers.go b/api/workers.go new file mode 100644 index 00000000..7ee5d7f1 --- /dev/null +++ b/api/workers.go @@ -0,0 +1,237 @@ +package api + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/krau/SaveAny-Bot/client/worker" + "github.com/krau/SaveAny-Bot/database" +) + +const maskedWorkerToken = "__MASKED__" + +type BotWorkerInput struct { + Name string `json:"name"` + Token string `json:"token"` + Enabled bool `json:"enabled"` +} + +type BotWorkerView struct { + ID uint `json:"id"` + Name string `json:"name"` + Enabled bool `json:"enabled"` + SessionPath string `json:"session_path,omitempty"` + TokenSet bool `json:"token_set"` + State string `json:"state"` + Username string `json:"username,omitempty"` + Error string `json:"error,omitempty"` +} + +func botWorkerView(record *database.BotWorker) BotWorkerView { + status := worker.StatusFor(record.ID) + return BotWorkerView{ + ID: record.ID, + Name: record.Name, + Enabled: record.Enabled, + SessionPath: record.SessionPath, + TokenSet: record.Token != "", + State: status.State, + Username: status.Username, + Error: status.Error, + } +} + +func (s *server) listBotWorkers(w http.ResponseWriter, r *http.Request) { + records, err := database.ListBotWorkers(r.Context()) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + out := make([]BotWorkerView, 0, len(records)) + for i := range records { + out = append(out, botWorkerView(&records[i])) + } + WriteJSON(w, http.StatusOK, map[string]any{"workers": out}) +} + +func (s *server) createBotWorker(w http.ResponseWriter, r *http.Request) { + var in BotWorkerInput + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + name := strings.TrimSpace(in.Name) + token := strings.TrimSpace(in.Token) + if name == "" || len([]rune(name)) > 64 { + WriteFieldErrors(w, http.StatusBadRequest, "invalid helper bot name", map[string]string{"name": "required, max 64 characters"}) + return + } + if token == "" { + WriteFieldErrors(w, http.StatusBadRequest, "token is required", map[string]string{"token": "required"}) + return + } + sessionPath := worker.NewSessionPath(s.deps.DataDir) + if err := os.MkdirAll(filepath.Dir(sessionPath), 0o755); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + record := &database.BotWorker{ + Name: name, + Token: token, + SessionPath: sessionPath, + Enabled: in.Enabled, + } + if err := database.CreateBotWorker(r.Context(), record); err != nil { + WriteError(w, http.StatusConflict, "worker_create_failed", err.Error()) + return + } + if record.Enabled { + if _, err := worker.StartWorker(s.deps.Ctx, record.ID); err != nil { + WriteError(w, http.StatusInternalServerError, "worker_start_failed", err.Error()) + return + } + } + WriteJSON(w, http.StatusCreated, botWorkerView(record)) +} + +func (s *server) updateBotWorker(w http.ResponseWriter, r *http.Request) { + id, ok := botWorkerID(w, r) + if !ok { + return + } + record, err := database.GetBotWorker(r.Context(), id) + if err != nil { + WriteError(w, http.StatusNotFound, "worker_not_found", "helper bot not found") + return + } + var in BotWorkerInput + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in); err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid request body") + return + } + name := strings.TrimSpace(in.Name) + if name == "" || len([]rune(name)) > 64 { + WriteFieldErrors(w, http.StatusBadRequest, "invalid helper bot name", map[string]string{"name": "required, max 64 characters"}) + return + } + fields := map[string]any{"name": name, "enabled": in.Enabled} + if token := strings.TrimSpace(in.Token); token != "" && token != maskedWorkerToken && token != record.Token { + worker.StopWorker(id) + if err := worker.RemoveSession(record.SessionPath); err != nil { + WriteError(w, http.StatusInternalServerError, "worker_session_cleanup_failed", err.Error()) + return + } + sessionPath := worker.NewSessionPath(s.deps.DataDir) + if err := os.MkdirAll(filepath.Dir(sessionPath), 0o755); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + fields["token"] = token + fields["session_path"] = sessionPath + } + if err := database.UpdateBotWorker(r.Context(), id, fields); err != nil { + WriteError(w, http.StatusConflict, "worker_update_failed", err.Error()) + return + } + record, err = database.GetBotWorker(r.Context(), id) + if err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if record.Enabled { + if _, err := worker.StartWorker(s.deps.Ctx, id); err != nil { + WriteError(w, http.StatusInternalServerError, "worker_start_failed", err.Error()) + return + } + } else { + worker.StopWorker(id) + } + WriteJSON(w, http.StatusOK, botWorkerView(record)) +} + +func (s *server) deleteBotWorker(w http.ResponseWriter, r *http.Request) { + id, ok := botWorkerID(w, r) + if !ok { + return + } + record, err := database.GetBotWorker(r.Context(), id) + if err != nil { + WriteError(w, http.StatusNotFound, "worker_not_found", "helper bot not found") + return + } + worker.StopWorker(id) + if err := database.DeleteBotWorker(r.Context(), id); err != nil { + WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + if err := worker.RemoveSession(record.SessionPath); err != nil { + WriteError(w, http.StatusInternalServerError, "worker_session_cleanup_failed", err.Error()) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *server) startBotWorker(w http.ResponseWriter, r *http.Request) { + id, ok := botWorkerID(w, r) + if !ok { + return + } + record, err := database.GetBotWorker(r.Context(), id) + if err != nil { + WriteError(w, http.StatusNotFound, "worker_not_found", "helper bot not found") + return + } + if !record.Enabled { + WriteError(w, http.StatusBadRequest, "worker_disabled", "helper bot is disabled") + return + } + status, err := worker.StartWorker(s.deps.Ctx, id) + if err != nil { + WriteError(w, http.StatusInternalServerError, "worker_start_failed", err.Error()) + return + } + WriteJSON(w, http.StatusOK, status) +} + +func (s *server) stopBotWorker(w http.ResponseWriter, r *http.Request) { + id, ok := botWorkerID(w, r) + if !ok { + return + } + if _, err := database.GetBotWorker(r.Context(), id); err != nil { + WriteError(w, http.StatusNotFound, "worker_not_found", "helper bot not found") + return + } + worker.StopWorker(id) + WriteJSON(w, http.StatusOK, worker.StatusFor(id)) +} + +func (s *server) testBotWorker(w http.ResponseWriter, r *http.Request) { + id, ok := botWorkerID(w, r) + if !ok { + return + } + if _, err := database.GetBotWorker(r.Context(), id); err != nil { + WriteError(w, http.StatusNotFound, "worker_not_found", "helper bot not found") + return + } + status := worker.StatusFor(id) + if status.State != worker.StateRunning { + WriteJSON(w, http.StatusOK, map[string]any{"ok": false, "error": status.Error, "state": status.State}) + return + } + WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "username": status.Username}) +} + +func botWorkerID(w http.ResponseWriter, r *http.Request) (uint, bool) { + id64, err := strconv.ParseUint(r.PathValue("id"), 10, 64) + if err != nil { + WriteError(w, http.StatusBadRequest, "invalid_request", "invalid helper bot id") + return 0, false + } + return uint(id64), true +} diff --git a/app/app.go b/app/app.go new file mode 100644 index 00000000..03b36b1b --- /dev/null +++ b/app/app.go @@ -0,0 +1,211 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/krau/mygotg/ext" + + "github.com/krau/SaveAny-Bot/api" + "github.com/krau/SaveAny-Bot/client/bot" + "github.com/krau/SaveAny-Bot/client/bot/handlers" + "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/client/worker" + "github.com/krau/SaveAny-Bot/common/selfupdate" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/core/resume" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/parsers" + "github.com/krau/SaveAny-Bot/settings" + "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/SaveAny-Bot/web" +) + +type Options struct { + DataDir string + DBPath string + Listen string + NoWeb bool +} + +type App struct { + ctx context.Context + cancel context.CancelFunc + svc *settings.Service + opts Options + web *web.Server + + applyMu sync.Mutex + mu sync.Mutex + resumeDone chan struct{} + appliedBotCred string + appliedUserbotSession string + flowMu sync.Mutex + flow *loginFlow +} + +func New(parent context.Context, svc *settings.Service, opts Options) *App { + ctx, cancel := context.WithCancel(parent) + return &App{ctx: ctx, cancel: cancel, svc: svc, opts: opts} +} + +func (a *App) Done() <-chan struct{} { return a.ctx.Done() } + +func (a *App) Start() error { + bot.SetRestartHandler(a.RequestRestart) + if err := database.MarkInterruptedTaskRecords(a.ctx); err != nil { + log.Errorf("Failed to mark interrupted tasks: %v", err) + } + worker.Init(a.ctx, a.opts.DataDir) + core.Run(a.ctx) + + cfg := a.svc.Config() + if cfg.Parser.PluginEnable { + dirs := cfg.Parser.PluginDirs + if len(dirs) == 0 { + dirs = []string{"plugins"} + } + for _, dir := range dirs { + if err := parsers.LoadPlugins(a.ctx, dir); err != nil { + log.Error("Failed to load parser plugins", "dir", dir, "error", err) + } + } + } + go handlers.ListenUserbotMediaEvents(a.ctx, func() *ext.Context { + if botCtx := bot.ExtContext(); botCtx != nil { + return botCtx + } + return user.GetCtx() + }) + + if !a.opts.NoWeb && cfg.Web.Enable { + webToken := strings.TrimSpace(cfg.Web.APIToken) + if webToken == "" { + webToken = config.GenerateToken() + } + a.web = web.New(a.opts.Listen, webToken, api.NewHandler(api.Deps{ + Ctx: a.ctx, + Settings: a.svc, + Telegram: a, + DataDir: a.opts.DataDir, + DBPath: a.opts.DBPath, + WebToken: webToken, + })) + if err := a.web.Start(a.ctx); err != nil { + return err + } + } + + go func() { + if err := a.ApplyTelegram(a.ctx); err != nil { + log.Errorf("Failed to apply telegram settings: %v", err) + } + }() + a.resumeDone = make(chan struct{}) + go func() { + defer close(a.resumeDone) + resume.Run(a.ctx) + }() + return nil +} + +func (a *App) ApplyTelegram(ctx context.Context) error { + a.applyMu.Lock() + defer a.applyMu.Unlock() + cfg := a.svc.Config() + cred := strings.Join([]string{ + cfg.Telegram.Token, + fmt.Sprintf("%d", cfg.Telegram.AppID), + fmt.Sprintf("%d", cfg.Telegram.RpcRetry), + cfg.Telegram.AppHash, + fmt.Sprintf("%t", cfg.Telegram.Proxy.Enable), + cfg.Telegram.Proxy.URL, + }, "|") + + userbotSession := cfg.Telegram.Userbot.Session + a.mu.Lock() + changed := cred != a.appliedBotCred + userbotSessionChanged := userbotSession != a.appliedUserbotSession + a.appliedBotCred = cred + a.appliedUserbotSession = userbotSession + a.mu.Unlock() + + botStatus := bot.CurrentStatus() + switch { + case cfg.Telegram.Token == "": + if botStatus.State != "stopped" { + bot.Stop() + } + case changed: + bot.Stop() + go func() { + if err := bot.Start(a.ctx); err != nil { + log.Errorf("Failed to start bot: %v", err) + } + }() + case botStatus.State == "stopped" || botStatus.State == "error": + go func() { + if err := bot.Start(a.ctx); err != nil { + log.Errorf("Failed to start bot: %v", err) + } + }() + } + + userStatus := user.CurrentStatus() + if userbotSessionChanged && (userStatus.State == "running" || userStatus.State == "starting") { + user.Stop() + userStatus = user.CurrentStatus() + } + if !cfg.Telegram.Userbot.Enable { + if userStatus.State == "running" || userStatus.State == "starting" { + user.Stop() + } + } else if userStatus.State == "stopped" || userStatus.State == "error" || userStatus.State == "login_required" { + go func() { + if err := user.Start(a.ctx, nil, cfg.Telegram.Userbot.Phone); err != nil && !errors.Is(err, user.ErrLoginRequired) { + log.Errorf("Failed to start userbot: %v", err) + } + }() + } + return nil +} + +func (a *App) RequestRestart() { + selfupdate.RequestRestart() + a.cancel() +} + +func (a *App) Stop() { + a.cancel() + if a.resumeDone != nil { + select { + case <-a.resumeDone: + case <-time.After(5 * time.Second): + log.Warn("Timed out waiting for queued task recovery to stop") + } + } + if a.web != nil { + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := a.web.Stop(stopCtx); err != nil { + log.Warnf("Failed to stop web server: %v", err) + } + cancel() + } + bot.Stop() + user.Stop() + worker.StopAll() + core.Close() + if !core.WaitWorkers(10 * time.Second) { + log.Warn("Timed out waiting for task workers to stop") + } + storage.CloseAll() + if err := database.Close(); err != nil { + log.Warnf("Failed to close database: %v", err) + } +} diff --git a/app/telegram.go b/app/telegram.go new file mode 100644 index 00000000..a9e73458 --- /dev/null +++ b/app/telegram.go @@ -0,0 +1,222 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/charmbracelet/log" + "github.com/rs/xid" + + "github.com/krau/SaveAny-Bot/api" + "github.com/krau/SaveAny-Bot/client/bot" + "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/settings" +) + +func (a *App) Status(ctx context.Context) api.TelegramStatus { + botStatus := bot.CurrentStatus() + userStatus := user.CurrentStatus() + cfg := a.svc.Config() + + status := api.TelegramStatus{ + Bot: api.BotStatus{ + State: botStatus.State, + ID: botStatus.ID, + Username: botStatus.Username, + Error: botStatus.Error, + StartedAt: botStatus.StartedAt, + }, + Userbot: api.UserbotStatus{ + Enabled: cfg.Telegram.Userbot.Enable, + State: userStatus.State, + ID: userStatus.ID, + Username: userStatus.Username, + Phone: userStatus.Phone, + Error: userStatus.Error, + }, + } + a.flowMu.Lock() + flow := a.flow + a.flowMu.Unlock() + if flow != nil { + state, errMsg := flow.status() + status.Userbot.Login = &api.LoginFlow{FlowID: flow.id, State: state, Error: errMsg} + } + return status +} + +func (a *App) StartBot(ctx context.Context) error { + if strings.TrimSpace(a.svc.Config().Telegram.Token) == "" { + return errors.New("bot token is not configured") + } + if bot.CurrentStatus().State == "running" || bot.CurrentStatus().State == "starting" { + return nil + } + go func() { + if err := bot.Start(a.ctx); err != nil { + log.Errorf("Failed to start bot: %v", err) + } + }() + return nil +} + +func (a *App) StopBot(ctx context.Context, force bool) error { + if !force && core.ActiveTaskCount() > 0 { + return errors.New("tasks are running") + } + bot.Stop() + return nil +} + +func (a *App) RestartBot(ctx context.Context, force bool) error { + if !force && core.ActiveTaskCount() > 0 { + return errors.New("tasks are running") + } + bot.Stop() + go func() { + if err := bot.Start(a.ctx); err != nil { + log.Errorf("Failed to restart bot: %v", err) + } + }() + return nil +} + +func (a *App) StartUserbotLogin(ctx context.Context, phone string) (string, error) { + if phone == "" { + return "", errors.New("phone number is required") + } + if user.CurrentStatus().State == "running" || user.CurrentStatus().State == "starting" { + return "", errors.New("userbot is already running") + } + cfg := a.svc.Config() + if cfg.Telegram.AppID == 0 || cfg.Telegram.AppHash == "" { + return "", errors.New("telegram app_id and app_hash are required") + } + + a.flowMu.Lock() + if a.flow != nil { + state, _ := a.flow.status() + if state != "failed" && state != "cancelled" && state != "done" { + a.flowMu.Unlock() + return "", errors.New("a login flow is already active") + } + } + flow := newLoginFlow(a.ctx, xid.New().String(), phone) + a.flow = flow + a.flowMu.Unlock() + + go func() { + err := user.Start(a.ctx, &webConversator{flow: flow}, phone) + if err != nil { + flow.markFailed(err) + return + } + if flow.ctx.Err() != nil { + user.Stop() + return + } + patch, err := json.Marshal(map[string]any{ + "userbot": map[string]any{"enable": true, "phone": phone}, + }) + if err != nil { + flow.markFailed(err) + user.Stop() + return + } + if _, err := a.svc.UpdateGroup(a.ctx, settings.GroupTelegram, patch); err != nil { + flow.markFailed(err) + user.Stop() + return + } + if !flow.markDone() { + user.Stop() + disable, err := json.Marshal(map[string]any{ + "userbot": map[string]any{"enable": false}, + }) + if err != nil { + log.Warnf("Failed to encode userbot disable patch: %v", err) + return + } + if _, err := a.svc.UpdateGroup(a.ctx, settings.GroupTelegram, disable); err != nil { + log.Warnf("Failed to disable userbot after login flow ended: %v", err) + } + return + } + flow.cancel() + }() + return flow.id, nil +} + +func (a *App) SubmitUserbotCode(ctx context.Context, flowID, code string) error { + flow, err := a.getFlow(flowID) + if err != nil { + return err + } + return flow.submitCode(code) +} + +func (a *App) SubmitUserbotPassword(ctx context.Context, flowID, password string) error { + flow, err := a.getFlow(flowID) + if err != nil { + return err + } + return flow.submitPassword(password) +} + +func (a *App) CancelUserbotLogin(ctx context.Context, flowID string) error { + flow, err := a.getFlow(flowID) + if err != nil { + return err + } + if state, _ := flow.status(); state == "done" { + return errors.New("login already completed") + } + flow.cancel() + flow.setState("cancelled", "") + user.Stop() + a.flowMu.Lock() + if a.flow == flow { + a.flow = nil + } + a.flowMu.Unlock() + return nil +} + +func (a *App) LogoutUserbot(ctx context.Context) error { + a.flowMu.Lock() + if a.flow != nil { + a.flow.cancel() + a.flow = nil + } + a.flowMu.Unlock() + if err := user.Logout(); err != nil { + return err + } + patch, err := json.Marshal(map[string]any{ + "userbot": map[string]any{"enable": false}, + }) + if err != nil { + return fmt.Errorf("encode userbot logout settings: %w", err) + } + if _, err := a.svc.UpdateGroup(ctx, settings.GroupTelegram, patch); err != nil { + return fmt.Errorf("failed to persist logout: %w", err) + } + return nil +} + +func (a *App) RestartProcess() { + a.RequestRestart() +} + +func (a *App) getFlow(flowID string) (*loginFlow, error) { + a.flowMu.Lock() + defer a.flowMu.Unlock() + if a.flow == nil || a.flow.id != flowID { + return nil, errors.New("unknown login flow") + } + return a.flow, nil +} diff --git a/app/userbot_auth.go b/app/userbot_auth.go new file mode 100644 index 00000000..7ea779bb --- /dev/null +++ b/app/userbot_auth.go @@ -0,0 +1,140 @@ +package app + +import ( + "context" + "fmt" + "sync" + "time" + + gotgproto "github.com/krau/mygotg" +) + +type loginFlow struct { + id string + phone string + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + state string + errMsg string + codeCh chan string + pwdCh chan string +} + +func newLoginFlow(parent context.Context, id, phone string) *loginFlow { + ctx, cancel := context.WithTimeout(parent, 5*time.Minute) + return &loginFlow{ + id: id, + phone: phone, + ctx: ctx, + cancel: cancel, + state: "starting", + codeCh: make(chan string, 1), + pwdCh: make(chan string, 1), + } +} + +func (f *loginFlow) setState(state, errMsg string) { + f.mu.Lock() + f.state = state + f.errMsg = errMsg + f.mu.Unlock() +} + +func (f *loginFlow) status() (string, string) { + f.mu.Lock() + defer f.mu.Unlock() + return f.state, f.errMsg +} + +func (f *loginFlow) markDone() bool { + f.mu.Lock() + defer f.mu.Unlock() + if f.state == "cancelled" { + return false + } + f.state = "done" + return true +} + +func (f *loginFlow) markFailed(err error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.state == "cancelled" { + return + } + f.state = "failed" + f.errMsg = err.Error() +} + +func (f *loginFlow) submitCode(code string) error { + state, _ := f.status() + if state != "code_required" { + return fmt.Errorf("login flow is not waiting for a code (state %s)", state) + } + select { + case f.codeCh <- code: + return nil + case <-f.ctx.Done(): + return f.ctx.Err() + } +} + +func (f *loginFlow) submitPassword(password string) error { + state, _ := f.status() + if state != "password_required" { + return fmt.Errorf("login flow is not waiting for a password (state %s)", state) + } + select { + case f.pwdCh <- password: + return nil + case <-f.ctx.Done(): + return f.ctx.Err() + } +} + +type webConversator struct { + flow *loginFlow +} + +func (c *webConversator) AskPhoneNumber() (string, error) { return c.flow.phone, nil } + +func (c *webConversator) AskCode() (string, error) { + c.flow.setState("code_required", "") + select { + case <-c.flow.ctx.Done(): + return "", c.flow.ctx.Err() + case code := <-c.flow.codeCh: + c.flow.setState("verifying", "") + return code, nil + } +} + +func (c *webConversator) AskPassword() (string, error) { + c.flow.setState("password_required", "") + select { + case <-c.flow.ctx.Done(): + return "", c.flow.ctx.Err() + case password := <-c.flow.pwdCh: + c.flow.setState("verifying", "") + return password, nil + } +} + +func (c *webConversator) AuthStatus(status gotgproto.AuthStatus) { + switch status.Event { + case gotgproto.AuthStatusPhoneCodeAsked: + c.flow.setState("code_required", "") + case gotgproto.AuthStatusPasswordAsked: + c.flow.setState("password_required", "") + case gotgproto.AuthStatusPhoneCodeRetrial, gotgproto.AuthStatusPasswordRetrial, gotgproto.AuthStatusPhoneRetrial: + c.flow.setState("code_required", fmt.Sprintf("incorrect credential, attempts left: %d", status.AttemptsLeft)) + case gotgproto.AuthStatusPhoneCodeFailed, gotgproto.AuthStatusPasswordFailed: + c.flow.setState("failed", "credential verification failed") + case gotgproto.AuthStatusSuccess: + // The flow is marked done only after the client is fully started and + // the enabled flag has been persisted. + c.flow.setState("verifying", "") + } +} diff --git a/boot/boot.go b/boot/boot.go new file mode 100644 index 00000000..5f9366f8 --- /dev/null +++ b/boot/boot.go @@ -0,0 +1,88 @@ +package boot + +import ( + "context" + "net/http" + "os" + "strings" + + "github.com/charmbracelet/log" + + "github.com/krau/SaveAny-Bot/common/cache" + "github.com/krau/SaveAny-Bot/common/i18n" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/parsers" + "github.com/krau/SaveAny-Bot/settings" +) + +type Result struct { + Config config.Config + Settings *settings.Service + DataDir string + DBPath string + Listen string + NoWeb bool +} + +func Init(ctx context.Context, opts Options) (Result, error) { + if err := os.MkdirAll(opts.DataDir, 0o700); err != nil { + return Result{}, err + } + if err := os.Chmod(opts.DataDir, 0o700); err != nil { + log.Warnf("Failed to restrict data directory permissions: %v", err) + } + database.Init(ctx, opts.DBPath) + if !settings.LegacyImported(ctx) { + if err := settings.ImportLegacy(ctx, opts.ConfigPath, opts.DataDir); err != nil { + log.Errorf("Failed to import legacy configuration: %v", err) + } + } + if err := database.MigrateMinioStorageType(ctx); err != nil { + log.Errorf("Failed to migrate MinIO storages to S3: %v", err) + } + svc, err := settings.NewService(ctx, opts.DataDir) + if err != nil { + return Result{}, err + } + if tempPath := strings.TrimSpace(svc.Config().Temp.BasePath); tempPath != "" { + if err := os.MkdirAll(tempPath, 0o755); err != nil { + return Result{}, err + } + } + if opts.Listen != "" { + svc.SetListenOverride(opts.Listen) + } + cfg := svc.Config() + if opts.InitWeb && !opts.NoWeb && cfg.Web.Enable { + // The generated password is emitted at Warn; force a level that shows it. + log.SetLevel(log.InfoLevel) + if err := svc.EnsureAdmin(os.Getenv("SAVEANY_ADMIN_PASSWORD")); err != nil { + return Result{}, err + } + } + if level, err := log.ParseLevel(strings.TrimSpace(cfg.Log.Level)); err == nil { + log.SetLevel(level) + } else { + log.Warn("Invalid log level, using info", "level", cfg.Log.Level) + } + i18n.Init(cfg.Lang) + parsers.Configure() + cache.Init() + if cfg.Proxy != "" { + transport, err := config.ProxyTransport(cfg.Proxy) + if err != nil { + log.Errorf("Invalid proxy configuration: %v", err) + } else { + http.DefaultTransport = transport + } + } + return Result{ + Config: cfg, + Settings: svc, + DataDir: opts.DataDir, + DBPath: opts.DBPath, + Listen: cfg.Web.Listen, + NoWeb: opts.NoWeb, + }, nil +} diff --git a/boot/flags.go b/boot/flags.go new file mode 100644 index 00000000..871c9195 --- /dev/null +++ b/boot/flags.go @@ -0,0 +1,73 @@ +package boot + +import ( + "os" + "path/filepath" + + "github.com/charmbracelet/log" + "github.com/spf13/cobra" +) + +func RegisterFlags(cmd *cobra.Command) { + flags := cmd.PersistentFlags() + flags.StringP("config", "c", "", "legacy config file path or URL (imported once)") + flags.String("data-dir", "", "data directory (default ./data)") + flags.String("db-path", "", "database file path (default /saveany.db)") + flags.String("listen", "", "web listen address as host:port (default 127.0.0.1:28080)") + flags.Bool("no-web", false, "do not start the web server") +} + +type Options struct { + ConfigPath string + DataDir string + DBPath string + Listen string + NoWeb bool + InitWeb bool +} + +func Parse(cmd *cobra.Command) Options { + flags := cmd.Flags() + getString := func(name string) string { + value, err := flags.GetString(name) + if err != nil { + log.Warnf("Failed to read --%s: %v", name, err) + } + return value + } + getBool := func(name string) bool { + value, err := flags.GetBool(name) + if err != nil { + log.Warnf("Failed to read --%s: %v", name, err) + } + return value + } + configPath := getString("config") + dataDir := getString("data-dir") + dbPath := getString("db-path") + listen := getString("listen") + noWeb := getBool("no-web") + + if dataDir == "" { + dataDir = os.Getenv("SAVEANY_DATA_DIR") + } + if dataDir == "" { + dataDir = "./data" + } + if dbPath == "" { + dbPath = os.Getenv("SAVEANY_DB_PATH") + } + if dbPath == "" { + dbPath = filepath.Join(dataDir, "saveany.db") + } + if listen == "" { + listen = os.Getenv("SAVEANY_LISTEN") + } + return Options{ + ConfigPath: configPath, + DataDir: dataDir, + DBPath: dbPath, + Listen: listen, + NoWeb: noWeb, + } +} diff --git a/client/bot/bot.go b/client/bot/bot.go index 71c1a1bb..d9e04ca3 100644 --- a/client/bot/bot.go +++ b/client/bot/bot.go @@ -2,43 +2,99 @@ package bot import ( "context" + "errors" + "sync" "time" - "github.com/celestix/gotgproto" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" - "github.com/celestix/gotgproto/sessionMaker" "github.com/charmbracelet/log" "github.com/gotd/td/tg" + gotgproto "github.com/krau/mygotg" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" + "github.com/krau/mygotg/session" + "github.com/krau/SaveAny-Bot/client/bot/handlers" "github.com/krau/SaveAny-Bot/client/middleware" "github.com/krau/SaveAny-Bot/common/i18n" + "github.com/krau/SaveAny-Bot/common/selfupdate" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" ) -var ectx *ext.Context +type Status struct { + State string + ID int64 + Username string + Error string + StartedAt time.Time +} + +type manager struct { + mu sync.RWMutex + client *gotgproto.Client + ectx *ext.Context + status Status + starting bool + generation uint64 + startCancel context.CancelFunc + restartHandler func() +} + +var mgr = &manager{status: Status{State: "stopped"}} +func SetRestartHandler(fn func()) { + mgr.mu.Lock() + mgr.restartHandler = fn + mgr.mu.Unlock() +} + +func CurrentStatus() Status { + mgr.mu.RLock() + defer mgr.mu.RUnlock() + return mgr.status +} + +// ExtContext returns the bot context, or nil when the bot is not running. func ExtContext() *ext.Context { - return ectx + mgr.mu.RLock() + defer mgr.mu.RUnlock() + return mgr.ectx } -func Init(ctx context.Context) <-chan struct{} { - log.FromContext(ctx).Info("Initializing Bot...") - resultChan := make(chan struct { +// Start connects the bot using the current configuration. It is idempotent +// while a client is already running. +func Start(ctx context.Context) error { + mgr.mu.Lock() + if mgr.client != nil { + mgr.mu.Unlock() + return nil + } + if mgr.starting { + mgr.mu.Unlock() + return errors.New("bot is already starting") + } + mgr.starting = true + mgr.generation++ + gen := mgr.generation + startCtx, cancelStart := context.WithCancel(ctx) + mgr.startCancel = cancelStart + mgr.status = Status{State: "starting"} + mgr.mu.Unlock() + + logger := log.FromContext(ctx) + logger.Info("Initializing Bot...") + + type result struct { client *gotgproto.Client err error - }) - shouldRestart := make(chan struct{}) + } + resultChan := make(chan result, 1) go func() { resolver, err := tgutil.NewConfigProxyResolver() if err != nil { - resultChan <- struct { - client *gotgproto.Client - err error - }{nil, err} + resultChan <- result{nil, err} return } client, err := gotgproto.NewClient( @@ -46,16 +102,22 @@ func Init(ctx context.Context) <-chan struct{} { config.C().Telegram.AppHash, gotgproto.ClientTypeBot(config.C().Telegram.Token), &gotgproto.ClientOpts{ - Session: sessionMaker.SqlSession(database.GetDialect(config.C().DB.Session)), - DisableCopyright: true, - Middlewares: middleware.NewDefaultMiddlewares(ctx, 5*time.Minute), - Resolver: resolver, - Context: ctx, - MaxRetries: config.C().Telegram.RpcRetry, - AutoFetchReply: true, + Session: session.SqlSession(database.GetDialect(config.C().DB.Session)), + DisableCopyright: true, + DeferUpdateRecovery: true, + Middlewares: middleware.NewDefaultMiddlewares(startCtx, 5*time.Minute), + Resolver: resolver, + Context: startCtx, + MaxRetries: config.C().Telegram.RpcRetry, + AutoFetchReply: true, ErrorHandler: func(ctx *ext.Context, u *ext.Update, s string) error { - if s == "SAVEANTBOT-RESTART" { - shouldRestart <- struct{}{} + if s == selfupdate.RestartSignal { + mgr.mu.RLock() + handler := mgr.restartHandler + mgr.mu.RUnlock() + if handler != nil { + handler() + } return dispatcher.EndGroups } log.FromContext(ctx).Errorf("unhandled error: %s", s) @@ -63,37 +125,105 @@ func Init(ctx context.Context) <-chan struct{} { }, }, ) - if err != nil { - resultChan <- struct { - client *gotgproto.Client - err error - }{nil, err} - return - } - commands := make([]tg.BotCommand, 0, len(handlers.CommandHandlers)) - for _, info := range handlers.CommandHandlers { - commands = append(commands, tg.BotCommand{Command: info.Cmd, Description: i18n.T(info.Desc)}) - } - _, err = client.API().BotsSetBotCommands(ctx, &tg.BotsSetBotCommandsRequest{ - Scope: &tg.BotCommandScopeDefault{}, - Commands: commands, - }) - resultChan <- struct { - client *gotgproto.Client - err error - }{client, err} + resultChan <- result{client, err} }() + var res result select { - case <-ctx.Done(): - log.FromContext(ctx).Errorf("Bot initialization cancelled: %s", ctx.Err()) - case result := <-resultChan: - if result.err != nil { - log.FromContext(ctx).Fatalf("Failed to initialize Bot: %s", result.err) + case <-startCtx.Done(): + mgr.mu.Lock() + if mgr.generation == gen { + mgr.starting = false + mgr.generation++ + mgr.startCancel = nil + mgr.status = Status{State: "stopped"} + } + mgr.mu.Unlock() + return startCtx.Err() + case res = <-resultChan: + } + + if res.err != nil { + mgr.mu.Lock() + if mgr.generation == gen { + mgr.starting = false + mgr.startCancel = nil + mgr.status = Status{State: "error", Error: res.err.Error()} + } + mgr.mu.Unlock() + return res.err + } + + mgr.mu.RLock() + stale := mgr.generation != gen || startCtx.Err() != nil + mgr.mu.RUnlock() + if stale { + res.client.Stop() + return errors.New("bot start canceled") + } + + ectx := res.client.CreateContext() + handlers.Register(res.client.Dispatcher) + if err := res.client.StartUpdateRecovery(startCtx); err != nil { + res.client.Stop() + mgr.mu.Lock() + if mgr.generation == gen { + mgr.starting = false + mgr.startCancel = nil + mgr.status = Status{State: "error", Error: err.Error()} } - ectx = result.client.CreateContext() - handlers.Register(result.client.Dispatcher, ectx) - log.FromContext(ctx).Info("Bot initialization completed.") + mgr.mu.Unlock() + return err + } + + commands := make([]tg.BotCommand, 0, len(handlers.CommandHandlers)) + for _, info := range handlers.CommandHandlers { + commands = append(commands, tg.BotCommand{Command: info.Cmd, Description: i18n.T(info.Desc)}) + } + if _, err := res.client.API().BotsSetBotCommands(startCtx, &tg.BotsSetBotCommandsRequest{ + Scope: &tg.BotCommandScopeDefault{}, + Commands: commands, + }); err != nil { + logger.Errorf("Failed to set bot commands: %v", err) + } + + status := Status{State: "running", StartedAt: time.Now()} + if res.client.Self != nil { + status.ID = res.client.Self.ID + status.Username = res.client.Self.Username + } + + mgr.mu.Lock() + if mgr.generation != gen { + mgr.mu.Unlock() + res.client.Stop() + return errors.New("bot start canceled") + } + mgr.client = res.client + mgr.ectx = ectx + mgr.starting = false + mgr.startCancel = nil + mgr.status = status + mgr.mu.Unlock() + + logger.Info("Bot initialization completed.") + return nil +} + +func Stop() { + mgr.mu.Lock() + client := mgr.client + mgr.client = nil + mgr.ectx = nil + mgr.starting = false + if mgr.startCancel != nil { + mgr.startCancel() + mgr.startCancel = nil + } + mgr.generation++ + mgr.status = Status{State: "stopped"} + mgr.mu.Unlock() + if client != nil { + client.Stop() } - return shouldRestart } diff --git a/client/bot/handlers/add_task.go b/client/bot/handlers/add_task.go index e8338a2f..ecdcf303 100644 --- a/client/bot/handlers/add_task.go +++ b/client/bot/handlers/add_task.go @@ -6,8 +6,6 @@ import ( "path" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" @@ -19,6 +17,8 @@ import ( "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "gorm.io/gorm" ) @@ -36,7 +36,7 @@ func handleAddCallback(ctx *ext.Context, update *ext.Update) error { msgID := update.CallbackQuery.GetMsgID() userID := update.CallbackQuery.GetUserID() - selectedStorage, err := storage.GetStorageByUserIDAndName(ctx, userID, data.SelectedStorName) + selectedStorage, err := storage.GetStorageByUserIDAndID(ctx, userID, data.SelectedStorageID) if err != nil { log.FromContext(ctx).Errorf("Failed to get storage: %s", err) ctx.AnswerCallback(msgelem.AlertCallbackAnswer(queryID, i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ @@ -44,7 +44,7 @@ func handleAddCallback(ctx *ext.Context, update *ext.Update) error { }))) return dispatcher.EndGroups } - dirs, err := database.GetDirsByUserChatIDAndStorageName(ctx, userID, data.SelectedStorName) + dirs, err := database.GetDirsByUserChatIDAndStorageID(ctx, userID, storage.ID(selectedStorage)) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { return fmt.Errorf("failed to get user directories: %w", err) } @@ -58,7 +58,7 @@ func handleAddCallback(ctx *ext.Context, update *ext.Update) error { }))) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgCommonPromptSelectDir, nil), ReplyMarkup: markup, @@ -83,31 +83,31 @@ func handleAddCallback(ctx *ext.Context, update *ext.Update) error { switch data.TaskType { case tasktype.TaskTypeTgfiles: if data.AsBatch { - return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userID, selectedStorage, dirPath, data.Files, msgID, data.ConflictStrategy) + return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userID, chatID(update), selectedStorage, dirPath, data.Files, msgID, data.ConflictStrategy) } - return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userID, selectedStorage, dirPath, data.Files[0], msgID, data.ConflictStrategy) + return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userID, chatID(update), selectedStorage, dirPath, data.Files[0], msgID, data.ConflictStrategy) case tasktype.TaskTypeTphpics: - return shortcut.CreateAndAddtelegraphWithEdit(ctx, userID, data.TphPageNode, data.TphDirPath, data.TphPics, selectedStorage, msgID) + return shortcut.CreateAndAddtelegraphWithEdit(ctx, chatID(update), data.TphPageNode, data.TphDirPath, data.TphPics, selectedStorage, msgID) case tasktype.TaskTypeParseditem: if len(data.ParsedItem.Resources) > 1 { dirPath = path.Join(dirPath, fsutil.NormalizePathname(data.ParsedItem.Title)) } - shortcut.CreateAndAddParsedTaskWithEdit(ctx, selectedStorage, dirPath, data.ParsedItem, msgID, userID) + shortcut.CreateAndAddParsedTaskWithEdit(ctx, selectedStorage, dirPath, data.ParsedItem, msgID, chatID(update)) case tasktype.TaskTypeDirectlinks: - shortcut.CreateAndAddDirectTaskWithEdit(ctx, selectedStorage, dirPath, data.DirectLinks, msgID, userID) + shortcut.CreateAndAddDirectTaskWithEdit(ctx, selectedStorage, dirPath, data.DirectLinks, msgID, chatID(update)) case tasktype.TaskTypeAria2: - client := GetAria2Client() - if client == nil { + client, err := GetAria2Client() + if err != nil { ctx.AnswerCallback(msgelem.AlertCallbackAnswer(queryID, i18n.T(i18nk.BotMsgAria2ErrorAria2ClientInitFailed, map[string]any{ - "Error": "aria2 client not initialized", + "Error": err.Error(), }))) return dispatcher.EndGroups } - shortcut.CreateAndAddAria2TaskWithEdit(ctx, selectedStorage, dirPath, data.Aria2URIs, client, msgID, userID) + shortcut.CreateAndAddAria2TaskWithEdit(ctx, selectedStorage, dirPath, data.Aria2URIs, client, msgID, chatID(update)) case tasktype.TaskTypeYtdlp: - shortcut.CreateAndAddYtdlpTaskWithEdit(ctx, selectedStorage, dirPath, data.YtdlpURLs, data.YtdlpFlags, msgID, userID) + shortcut.CreateAndAddYtdlpTaskWithEdit(ctx, selectedStorage, dirPath, data.YtdlpURLs, data.YtdlpFlags, msgID, chatID(update)) case tasktype.TaskTypeTransfer: - return handleTransferCallback(ctx, userID, selectedStorage, dirPath, data, msgID) + return handleTransferCallback(ctx, userID, chatID(update), selectedStorage, dirPath, data, msgID) default: return fmt.Errorf("unexcept task type: %s", data.TaskType) } diff --git a/client/bot/handlers/cancel_task.go b/client/bot/handlers/cancel_task.go index e3a03c1b..eef1b540 100644 --- a/client/bot/handlers/cancel_task.go +++ b/client/bot/handlers/cancel_task.go @@ -4,14 +4,14 @@ import ( "fmt" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/core" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleCancelCallback(ctx *ext.Context, update *ext.Update) error { @@ -28,31 +28,10 @@ func handleCancelCallback(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } - ctx.EditMessage(update.CallbackQuery.GetUserID(), &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgCancelInfoCancellingTask, nil), }) return dispatcher.EndGroups } - -func handleCancelCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - args := strings.Fields(update.EffectiveMessage.Text) - if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCancelUsage, nil)), nil) - return dispatcher.EndGroups - } - taskID := args[1] - if err := core.CancelTask(ctx, taskID); err != nil { - logger.Errorf("failed to cancel task %s: %v", taskID, err) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCancelErrorCancelFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return dispatcher.EndGroups - } - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCancelInfoCancelRequested, map[string]any{ - "TaskID": taskID, - })), nil) - return dispatcher.EndGroups -} diff --git a/client/bot/handlers/config.go b/client/bot/handlers/config.go index c8c363dc..343cb1a8 100644 --- a/client/bot/handlers/config.go +++ b/client/bot/handlers/config.go @@ -5,40 +5,52 @@ import ( "strings" "text/template" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" + "github.com/gotd/td/telegram/message/markup" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/conflictutil" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/enums/fnamest" "github.com/krau/SaveAny-Bot/pkg/tcbdata" + "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleConfigCmd(ctx *ext.Context, update *ext.Update) error { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgConfigPromptSelectOption)), &ext.ReplyOpts{ - Markup: &tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - &tg.KeyboardButtonCallback{ - Text: i18n.T(i18nk.BotMsgConfigButtonFilenameStrategy), - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeConfig, "fnamest"), - }, - &tg.KeyboardButtonCallback{ - Text: i18n.T(i18nk.BotMsgConfigButtonConflictStrategy), - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeConfig, "conflictst"), - }, - }, - }, - }, - }, + ctx.Reply(update, ext.ReplyTextString(configMenuText("")), &ext.ReplyOpts{ + Markup: configMenuMarkup(), }) return dispatcher.EndGroups } +func configMenuText(extra string) string { + text := i18n.T(i18nk.BotMsgConfigMenuTitle) + "\n\n" + i18n.T(i18nk.BotMsgConfigMenuHint) + if extra != "" { + text += "\n\n" + extra + } + return text +} + +func configMenuMarkup() *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{ + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonFilenameStrategy), fmt.Appendf(nil, "%s fnamest", tcbdata.TypeConfig)), + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonConflictStrategy), fmt.Appendf(nil, "%s conflictst", tcbdata.TypeConfig)), + }}, + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonSilent), fmt.Appendf(nil, "%s silent", tcbdata.TypeConfig)), + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonDefaultStorage), fmt.Appendf(nil, "%s storage", tcbdata.TypeConfig)), + }}, + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonTemplate), fmt.Appendf(nil, "%s template", tcbdata.TypeConfig)), + }}, + }} +} + func handleConfigCallback(ctx *ext.Context, update *ext.Update) error { args := strings.Fields(string(update.CallbackQuery.Data)) invaildDataAnswer := func() error { @@ -54,15 +66,101 @@ func handleConfigCallback(ctx *ext.Context, update *ext.Update) error { return invaildDataAnswer() } switch args[1] { + case "menu": + return handleConfigMenuCallback(ctx, update) case "fnamest": return handleConfigFnameSTCallback(ctx, update) case "conflictst": return handleConfigConflictSTCallback(ctx, update) + case "silent": + return handleConfigSilentCallback(ctx, update) + case "storage": + return handleConfigStorageCallback(ctx, update) + case "template": + return handleConfigTemplateCallback(ctx, update) default: return invaildDataAnswer() } } +func handleConfigMenuCallback(ctx *ext.Context, update *ext.Update) error { + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: configMenuText(""), + ReplyMarkup: configMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func handleConfigSilentCallback(ctx *ext.Context, update *ext.Update) error { + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + if !user.Silent && user.DefaultStorageID == 0 { + ctx.AnswerCallback(msgelem.AlertCallbackAnswer(update.CallbackQuery.GetQueryID(), i18n.T(i18nk.BotMsgCommonErrorDefaultStorageNotSet))) + return dispatcher.EndGroups + } + user.Silent = !user.Silent + if err := database.UpdateUser(ctx, user); err != nil { + return err + } + state := i18n.T(i18nk.BotMsgCommonInfoSilentModeOff) + if user.Silent { + state = i18n.T(i18nk.BotMsgCommonInfoSilentModeOn) + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: configMenuText(state), + ReplyMarkup: configMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func handleConfigStorageCallback(ctx *ext.Context, update *ext.Update) error { + userID := update.CallbackQuery.GetUserID() + storages := storage.GetUserStorages(ctx, userID) + if len(storages) == 0 { + ctx.AnswerCallback(msgelem.AlertCallbackAnswer(update.CallbackQuery.GetQueryID(), i18n.T(i18nk.BotMsgCommonErrorNoAvailableStorage))) + return dispatcher.EndGroups + } + markup, err := msgelem.BuildSetDefaultStorageMarkup(ctx, storages) + if err != nil { + return err + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: i18n.T(i18nk.BotMsgCommonPromptSelectDefaultStorage), + ReplyMarkup: markup, + }) + return dispatcher.EndGroups +} + +func handleConfigTemplateCallback(ctx *ext.Context, update *ext.Update) error { + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + text := i18n.T(i18nk.BotMsgConfigTemplateHelp) + if user.FilenameTemplate != "" { + text += "\n\n" + i18n.T(i18nk.BotMsgConfigInfoCurrentTemplatePrefix, map[string]any{ + "Template": user.FilenameTemplate, + }) + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: text, + ReplyMarkup: &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{{ + Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgConfigButtonBack), fmt.Appendf(nil, "%s menu", tcbdata.TypeConfig)), + }, + }}}, + }) + return dispatcher.EndGroups +} + func handleConfigFnameSTCallback(ctx *ext.Context, update *ext.Update) error { userID := update.CallbackQuery.GetUserID() user, err := database.GetUserByChatID(ctx, userID) @@ -80,7 +178,7 @@ func handleConfigFnameSTCallback(ctx *ext.Context, update *ext.Update) error { if err := database.UpdateUser(ctx, user); err != nil { return err } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgConfigInfoFilenameStrategySet, map[string]any{ "Strategy": fnamest.GetDisplay(st, config.C().Lang), @@ -89,14 +187,14 @@ func handleConfigFnameSTCallback(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } opts := fnamest.FnameSTValues() - buttons := make([]tg.KeyboardButtonClass, 0, len(opts)) + buttons := make([]tg.KeyboardInlineButton, 0, len(opts)) for _, opt := range opts { - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: fnamest.GetDisplay(opt, config.C().Lang), - Data: fmt.Appendf(nil, "%s %s %s", tcbdata.TypeConfig, "fnamest", opt), - }) + buttons = append(buttons, markup.Callback( + fnamest.GetDisplay(opt, config.C().Lang), + fmt.Appendf(nil, "%s %s %s", tcbdata.TypeConfig, "fnamest", opt), + )) } - markup := &tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{ + inlineKeyboard := &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{ {Buttons: buttons}, }} currentStStr := user.FilenameStrategy @@ -107,12 +205,12 @@ func handleConfigFnameSTCallback(ctx *ext.Context, update *ext.Update) error { if err != nil { currentSt = fnamest.Default } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgConfigPromptSelectFilenameStrategy, map[string]any{ "Strategy": fnamest.GetDisplay(currentSt, config.C().Lang), }), - ReplyMarkup: markup, + ReplyMarkup: inlineKeyboard, }) return dispatcher.EndGroups } @@ -133,7 +231,7 @@ func handleConfigConflictSTCallback(ctx *ext.Context, update *ext.Update) error if err := database.UpdateUser(ctx, user); err != nil { return err } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgConfigInfoConflictStrategySet, map[string]any{ "Strategy": conflictutil.Display(selected), @@ -143,38 +241,38 @@ func handleConfigConflictSTCallback(ctx *ext.Context, update *ext.Update) error } opts := tcbdata.ConflictStrategyValues() - rows := make([]tg.KeyboardButtonRow, 0, len(opts)) + rows := make([]tg.KeyboardInlineButtonRow, 0, len(opts)) for _, opt := range opts { - rows = append(rows, tg.KeyboardButtonRow{ - Buttons: []tg.KeyboardButtonClass{ - &tg.KeyboardButtonCallback{ - Text: conflictutil.Display(opt), - Data: fmt.Appendf(nil, "%s %s %s", tcbdata.TypeConfig, "conflictst", opt), - }, + rows = append(rows, tg.KeyboardInlineButtonRow{ + Buttons: []tg.KeyboardInlineButton{ + markup.Callback( + conflictutil.Display(opt), + fmt.Appendf(nil, "%s %s %s", tcbdata.TypeConfig, "conflictst", opt), + ), }, }) } - markup := &tg.ReplyInlineMarkup{Rows: rows} + inlineKeyboard := &tg.ReplyInlineMarkup{Rows: rows} currentSt := conflictutil.EffectiveStrategy(user) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgConfigPromptSelectConflictStrategy, map[string]any{ "Strategy": conflictutil.Display(currentSt), }), - ReplyMarkup: markup, + ReplyMarkup: inlineKeyboard, }) return dispatcher.EndGroups } -func handleConfigFnameTmpl(ctx *ext.Context, update *ext.Update) error { - userID := update.GetUserChat().GetID() +func handleTemplateCmd(ctx *ext.Context, update *ext.Update) error { + userID := responsibleUserID(update) user, err := database.GetUserByChatID(ctx, userID) if err != nil { return err } args := strings.Fields(string(update.EffectiveMessage.Text)) if len(args) <= 1 { - text := i18n.T(i18nk.BotMsgConfigFnametmplHelp, nil) + text := i18n.T(i18nk.BotMsgConfigTemplateHelp, nil) if user.FilenameTemplate != "" { text += "\n\n" + i18n.T(i18nk.BotMsgConfigInfoCurrentTemplatePrefix, map[string]any{ "Template": user.FilenameTemplate, diff --git a/client/bot/handlers/dir.go b/client/bot/handlers/dir.go index 85058eb1..3f65bda1 100644 --- a/client/bot/handlers/dir.go +++ b/client/bot/handlers/dir.go @@ -1,24 +1,30 @@ package handlers import ( + "fmt" "strconv" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/message/markup" + "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/utils/strutil" "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleDirCmd(ctx *ext.Context, update *ext.Update) error { logger := log.FromContext(ctx) - args := strings.Split(update.EffectiveMessage.Text, " ") - userChatID := update.GetUserChat().GetID() + args := strutil.ParseArgsRespectQuotes(update.EffectiveMessage.Text) + userChatID := responsibleUserID(update) dirs, err := database.GetUserDirsByChatID(ctx, userChatID) if err != nil { logger.Errorf("Failed to get user directories: %s", err) @@ -26,37 +32,38 @@ func handleDirCmd(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildDirHelpStyling(dirs)), nil) + ctx.Reply(update, ext.ReplyTextString(dirMenuText(dirs)), &ext.ReplyOpts{Markup: dirMenuMarkup(dirs)}) return dispatcher.EndGroups } - user, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID()) + user, err := database.GetUserByChatID(ctx, responsibleUserID(update)) if err != nil { logger.Errorf("Failed to get user: %s", err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirErrorGetUserFailed)), nil) return dispatcher.EndGroups } switch args[1] { - case "add": - // /dir add local1 path/to/dir + case "add", "new": + // /dirs add local1 path/to/dir if len(args) < 4 { ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildDirHelpStyling(dirs)), nil) return dispatcher.EndGroups } - if _, err := storage.GetStorageByUserIDAndName(ctx, user.ChatID, args[2]); err != nil { + selectedStorage, err := storage.GetStorageByUserIDAndName(ctx, user.ChatID, args[2]) + if err != nil { logger.Errorf("Failed to get storage %q: %s", args[2], err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, tgutil.EscapeHTMLTemplateData(map[string]any{"Error": err.Error()}))), nil) return dispatcher.EndGroups } - if err := database.CreateDirForUser(ctx, user.ID, args[2], args[3]); err != nil { + if err := database.CreateDirForUser(ctx, user.ID, storage.ID(selectedStorage), args[3]); err != nil { logger.Errorf("Failed to create directory: %s", err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirErrorCreateDirFailed)), nil) return dispatcher.EndGroups } ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirInfoCreateDirSuccess)), nil) - case "del": - // /dir del 3 + case "del", "remove": + // /dirs del 3 if len(args) < 3 { ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildDirHelpStyling(dirs)), nil) return dispatcher.EndGroups @@ -66,14 +73,76 @@ func handleDirCmd(ctx *ext.Context, update *ext.Update) error { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirErrorInvalidDirId)), nil) return dispatcher.EndGroups } - if err := database.DeleteDirByID(ctx, uint(dirID)); err != nil { + if err := database.DeleteDirByID(ctx, uint(dirID), user.ID); err != nil { logger.Errorf("Failed to delete directory: %s", err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirErrorDeleteDirFailed)), nil) return dispatcher.EndGroups } ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirInfoDeleteDirSuccess)), nil) default: - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDirErrorUnknownOperation)), nil) + ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildDirHelpStyling(dirs)), nil) + } + return dispatcher.EndGroups +} + +func dirMenuText(dirs []database.Dir) string { + if len(dirs) == 0 { + return i18n.T(i18nk.BotMsgDirMenuEmpty) + "\n\n" + i18n.T(i18nk.BotMsgDirMenuHint) + } + var b strings.Builder + b.WriteString(i18n.T(i18nk.BotMsgDirMenuTitle)) + b.WriteString("\n\n") + b.WriteString(i18n.T(i18nk.BotMsgDirMenuHint)) + for _, dir := range dirs { + storageName := config.C().GetStorageNameByID(dir.StorageID) + if storageName == "" { + storageName = fmt.Sprintf("#%d", dir.StorageID) + } + fmt.Fprintf(&b, "\n\n#%d %s - %s", dir.ID, storageName, dir.Path) + } + return b.String() +} + +func dirMenuMarkup(dirs []database.Dir) *tg.ReplyInlineMarkup { + if len(dirs) == 0 { + return nil + } + rows := make([]tg.KeyboardInlineButtonRow, 0, len(dirs)) + for _, dir := range dirs { + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgDirButtonDelete), fmt.Appendf(nil, "%s del %d", tcbdata.TypeDir, dir.ID)), + }}) + } + return &tg.ReplyInlineMarkup{Rows: rows} +} + +func handleDirCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + if args[1] == "del" && len(args) >= 3 { + dirID, err := strconv.Atoi(args[2]) + if err != nil { + return err + } + if err := database.DeleteDirByID(ctx, uint(dirID), user.ID); err != nil { + return err + } + } + dirs, err := database.GetUserDirsByChatID(ctx, userID) + if err != nil { + return err } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: dirMenuText(dirs), + ReplyMarkup: dirMenuMarkup(dirs), + }) return dispatcher.EndGroups } diff --git a/client/bot/handlers/dl.go b/client/bot/handlers/dl.go deleted file mode 100644 index 29664066..00000000 --- a/client/bot/handlers/dl.go +++ /dev/null @@ -1,112 +0,0 @@ -package handlers - -import ( - "net/url" - "strings" - "sync" - - "github.com/celestix/gotgproto/ext" - "github.com/charmbracelet/log" - "github.com/duke-git/lancet/v2/slice" - "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" - "github.com/krau/SaveAny-Bot/common/i18n" - "github.com/krau/SaveAny-Bot/common/i18n/i18nk" - "github.com/krau/SaveAny-Bot/config" - "github.com/krau/SaveAny-Bot/pkg/aria2" - "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" - "github.com/krau/SaveAny-Bot/pkg/tcbdata" - "github.com/krau/SaveAny-Bot/storage" -) - -func handleDlCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - args := strings.Split(update.EffectiveMessage.Text, " ") - if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDlUsage)), nil) - return nil - } - links := args[1:] - for i, link := range links { - links[i] = strings.TrimSpace(link) - u, err := url.Parse(link) - if err != nil || u.Scheme == "" || u.Host == "" { - logger.Warn("invaild link", link) - links[i] = "" - } - } - links = slice.Compact(links) - if len(links) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDlErrorNoValidLinks)), nil) - return nil - } - markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, update.GetUserChat().GetID()), tcbdata.Add{ - TaskType: tasktype.TaskTypeDirectlinks, - DirectLinks: links, - }) - if err != nil { - return err - } - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDlInfoFilesSelectStorage, map[string]any{ - "Count": len(links), - })), &ext.ReplyOpts{ - Markup: markup, - }) - return nil -} - -var aria2ClientInitOnce sync.Once -var aria2ClientInitErr error -var aria2Client *aria2.Client - -func GetAria2Client() *aria2.Client { - return aria2Client -} - -func handleAria2DlCmd(ctx *ext.Context, update *ext.Update) error { - if !config.C().Aria2.Enable { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgAria2ErrorAria2NotEnabled)), nil) - return nil - } - logger := log.FromContext(ctx) - args := strings.Split(update.EffectiveMessage.Text, " ") - if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDlUsage)), nil) - return nil - } - links := args[1:] - for i, link := range links { - links[i] = strings.TrimSpace(link) - } - links = slice.Compact(links) - if len(links) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDlErrorNoValidLinks)), nil) - return nil - } - logger.Debug("Preparing aria2 download", "links", links) - - // Initialize aria2 client to check connection - aria2ClientInitOnce.Do(func() { - aria2Client, aria2ClientInitErr = aria2.NewClient(config.C().Aria2.Url, config.C().Aria2.Secret) - }) - if aria2ClientInitErr != nil { - logger.Error("Failed to initialize aria2 client", "error", aria2ClientInitErr) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgAria2ErrorAria2ClientInitFailed, map[string]any{ - "Error": aria2ClientInitErr.Error(), - })), nil) - return nil - } - - // don't add to aria2 yet - markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, update.GetUserChat().GetID()), tcbdata.Add{ - TaskType: tasktype.TaskTypeAria2, - Aria2URIs: links, - }) - if err != nil { - return err - } - - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgAria2InfoSelectStorage)), &ext.ReplyOpts{ - Markup: markup, - }) - return nil -} diff --git a/client/bot/handlers/download.go b/client/bot/handlers/download.go new file mode 100644 index 00000000..f4fa11fc --- /dev/null +++ b/client/bot/handlers/download.go @@ -0,0 +1,191 @@ +package handlers + +import ( + "fmt" + "net/url" + "strings" + "sync" + + "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/message/markup" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut" + "github.com/krau/SaveAny-Bot/common/cache" + "github.com/krau/SaveAny-Bot/common/i18n" + "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/aria2" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" + "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" + "github.com/rs/xid" +) + +type downloadOption struct { + label i18nk.Key + data tcbdata.Add +} + +// handleDownloadCmd parses links once, then lets the user choose the download +// engine selection. +func handleDownloadCmd(ctx *ext.Context, update *ext.Update) error { + logger := log.FromContext(ctx) + args := splitQuotedArgs(update.EffectiveMessage.Text) + if len(args) < 2 { + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDownloadUsage)), nil) + return dispatcher.EndGroups + } + + webLinks, flags := parseYtdlpArgs(logger, args[1:]) + allLinks := parseDownloadURIs(args[1:], webLinks) + if len(webLinks) == 0 && len(allLinks) == 0 { + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDownloadErrorNoValidLinks)), nil) + return dispatcher.EndGroups + } + + options := make([]downloadOption, 0, 3) + if len(webLinks) > 0 && len(flags) == 0 { + options = append(options, downloadOption{ + label: i18nk.BotMsgDownloadButtonDirect, + data: tcbdata.Add{ + TaskType: tasktype.TaskTypeDirectlinks, + DirectLinks: webLinks, + }, + }) + } + if config.C().Aria2.Enable && len(allLinks) > 0 { + options = append(options, downloadOption{ + label: i18nk.BotMsgDownloadButtonAria2, + data: tcbdata.Add{ + TaskType: tasktype.TaskTypeAria2, + Aria2URIs: allLinks, + }, + }) + } + if len(webLinks) > 0 { + options = append(options, downloadOption{ + label: i18nk.BotMsgDownloadButtonVideo, + data: tcbdata.Add{ + TaskType: tasktype.TaskTypeYtdlp, + YtdlpURLs: webLinks, + YtdlpFlags: flags, + }, + }) + } + if len(options) == 0 { + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDownloadErrorNoEngine)), nil) + return dispatcher.EndGroups + } + + buttons := make([]tg.KeyboardInlineButton, 0, len(options)) + for _, option := range options { + dataID := xid.New().String() + if err := cache.Set(dataID, option.data); err != nil { + return err + } + buttons = append(buttons, markup.Callback( + i18n.T(option.label, nil), + fmt.Appendf(nil, "%s %s", tcbdata.TypeEngine, dataID), + )) + } + markup := &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{{Buttons: buttons}}} + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgDownloadPromptSelectEngine, map[string]any{ + "Count": len(allLinks), + })), &ext.ReplyOpts{Markup: markup}) + return dispatcher.EndGroups +} + +func parseDownloadURIs(args, webLinks []string) []string { + links := make([]string, 0, len(args)) + seen := make(map[string]struct{}, len(args)) + add := func(link string) { + if link == "" { + return + } + if _, ok := seen[link]; ok { + return + } + seen[link] = struct{}{} + links = append(links, link) + } + for _, link := range webLinks { + add(link) + } + for _, arg := range args { + if arg == "" || strings.HasPrefix(arg, "-") { + continue + } + u, err := url.Parse(arg) + if err == nil && u.Scheme != "" { + add(arg) + } + } + return links +} + +func handleEngineCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } + data, err := shortcut.GetCallbackDataWithAnswer[tcbdata.Add](ctx, update, args[1]) + if err != nil { + return err + } + storages := storage.GetUserStorages(ctx, update.CallbackQuery.GetUserID()) + if len(storages) == 0 { + ctx.AnswerCallback(msgelem.AlertCallbackAnswer(update.CallbackQuery.GetQueryID(), i18n.T(i18nk.BotMsgCommonErrorNoAvailableStorage))) + return dispatcher.EndGroups + } + markup, err := msgelem.BuildAddSelectStorageKeyboard(storages, data) + if err != nil { + return err + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: i18n.T(i18nk.BotMsgDownloadInfoSelectStorage, map[string]any{"Count": downloadOptionCount(data)}), + ReplyMarkup: markup, + }) + return dispatcher.EndGroups +} + +func downloadOptionCount(data tcbdata.Add) int { + switch data.TaskType { + case tasktype.TaskTypeDirectlinks: + return len(data.DirectLinks) + case tasktype.TaskTypeAria2: + return len(data.Aria2URIs) + case tasktype.TaskTypeYtdlp: + return len(data.YtdlpURLs) + default: + return 0 + } +} + +var aria2ClientCache struct { + sync.Mutex + key string + client *aria2.Client +} + +// GetAria2Client returns a client for the current aria2 settings. The cache is +// keyed by URL and secret so changing settings does not require a restart. +func GetAria2Client() (*aria2.Client, error) { + cfg := config.C().Aria2 + key := cfg.Url + "\x00" + cfg.Secret + aria2ClientCache.Lock() + defer aria2ClientCache.Unlock() + if aria2ClientCache.client != nil && aria2ClientCache.key == key { + return aria2ClientCache.client, nil + } + client, err := aria2.NewClient(cfg.Url, cfg.Secret) + if err != nil { + return nil, err + } + aria2ClientCache.client = client + aria2ClientCache.key = key + return client, nil +} diff --git a/client/bot/handlers/help.go b/client/bot/handlers/help.go index 0f18e3bc..2071d235 100644 --- a/client/bot/handlers/help.go +++ b/client/bot/handlers/help.go @@ -3,11 +3,11 @@ package handlers import ( "fmt" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleHelpCmd(ctx *ext.Context, update *ext.Update) error { diff --git a/client/bot/handlers/link.go b/client/bot/handlers/link.go index d4369b1e..969cc4a6 100644 --- a/client/bot/handlers/link.go +++ b/client/bot/handlers/link.go @@ -1,8 +1,6 @@ package handlers import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/dirutil" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" @@ -11,6 +9,8 @@ import ( "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleMessageLink(ctx *ext.Context, update *ext.Update) error { @@ -19,7 +19,7 @@ func handleMessageLink(ctx *ext.Context, update *ext.Update) error { return err } logger := log.FromContext(ctx) - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) stors := storage.GetUserStorages(ctx, userId) if len(files) == 1 { req, err := msgelem.BuildAddOneSelectStorageMessage(ctx, stors, files[0], replied.ID) @@ -55,9 +55,9 @@ func handleSilentSaveLink(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) if len(files) == 1 { - return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userId, stor, dirutil.PathFromContext(ctx), files[0], replied.ID) + return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userId, chatID(update), stor, dirutil.PathFromContext(ctx), files[0], replied.ID) } - return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userId, stor, dirutil.PathFromContext(ctx), files, replied.ID) + return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userId, chatID(update), stor, dirutil.PathFromContext(ctx), files, replied.ID) } diff --git a/client/bot/handlers/media.go b/client/bot/handlers/media.go index 6a7e554a..73107d55 100644 --- a/client/bot/handlers/media.go +++ b/client/bot/handlers/media.go @@ -1,8 +1,6 @@ package handlers import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/dirutil" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" @@ -12,6 +10,8 @@ import ( "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleMediaMessage(ctx *ext.Context, update *ext.Update) error { @@ -22,7 +22,7 @@ func handleMediaMessage(ctx *ext.Context, update *ext.Update) error { return handleGroupMediaMessage(ctx, update, message, groupID) } logger.Debugf("Got media: %s", message.Media.TypeName()) - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) userDB, err := database.GetUserByChatID(ctx, userId) if err != nil { return err @@ -55,7 +55,7 @@ func handleSilentSaveMedia(ctx *ext.Context, update *ext.Update) error { return handleGroupMediaMessage(ctx, update, message, groupID) } logger.Debugf("Got media: %s", message.Media.TypeName()) - userID := update.GetUserChat().GetID() + userID := responsibleUserID(update) userDB, err := database.GetUserByChatID(ctx, userID) if err != nil { return err @@ -65,5 +65,5 @@ func handleSilentSaveMedia(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userID, stor, dirutil.PathFromContext(ctx), file, msg.ID) + return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userID, chatID(update), stor, dirutil.PathFromContext(ctx), file, msg.ID) } diff --git a/client/bot/handlers/media_group.go b/client/bot/handlers/media_group.go index 8230ddd8..9f212367 100644 --- a/client/bot/handlers/media_group.go +++ b/client/bot/handlers/media_group.go @@ -4,8 +4,6 @@ import ( "sync" "time" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" @@ -13,11 +11,14 @@ import ( "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/shortcut" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/mediafilter" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) // mediaGroupKey uniquely identifies a media group by chat, sender, and group @@ -57,11 +58,10 @@ func handleGroupMediaMessage(ctx *ext.Context, update *ext.Update, message *tg.M mediaGroupHandler.SetupTimeout(max(config.C().Telegram.MediaGroupTimeout, 1)) logger := log.FromContext(ctx) media := message.Media - supported := mediautil.IsSupported(media) - if !supported { + if !mediautil.IsSupported(media) || !mediafilter.Allowed(media) { return dispatcher.EndGroups } - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) userDB, err := database.GetUserByChatID(ctx, userId) if err != nil { return err @@ -106,7 +106,7 @@ func processMediaGroup(ctx *ext.Context, update *ext.Update, key mediaGroupKey) } logger.Debugf("Processing media group %d with %d items", key.groupID, len(items)) - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) msg, err := ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgMediaGroupInfoSavingFiles, nil)), nil) if err != nil { logger.Errorf("Failed to reply: %s", err) @@ -116,10 +116,10 @@ func processMediaGroup(ctx *ext.Context, update *ext.Update, key mediaGroupKey) if stor != nil { // In silent mode if len(items) == 1 { - shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userId, stor, "", items[0], msg.ID) + shortcut.CreateAndAddTGFileTaskWithEdit(ctx, userId, key.chatID, stor, "", items[0], msg.ID) return } - shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userId, stor, "", items, msg.ID) + shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, userId, key.chatID, stor, "", items, msg.ID) return } @@ -130,7 +130,7 @@ func processMediaGroup(ctx *ext.Context, update *ext.Update, key mediaGroupKey) }) if err != nil { logger.Errorf("Failed to build storage selection keyboard: %s", err) - ctx.EditMessage(userId, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(key.chatID, &tg.MessagesEditMessageRequest{ ID: msg.ID, Message: i18n.T(i18nk.BotMsgMediaGroupErrorBuildStorageSelectKeyboardFailed, map[string]any{ "Error": err.Error(), @@ -138,7 +138,7 @@ func processMediaGroup(ctx *ext.Context, update *ext.Update, key mediaGroupKey) }) return } - ctx.EditMessage(userId, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(key.chatID, &tg.MessagesEditMessageRequest{ ID: msg.ID, Message: i18n.T(i18nk.BotMsgMediaGroupInfoGroupFoundFilesSelectStorage, map[string]any{ "Count": len(items), diff --git a/client/bot/handlers/middleware.go b/client/bot/handlers/middleware.go index 678db352..0e3f30f0 100644 --- a/client/bot/handlers/middleware.go +++ b/client/bot/handlers/middleware.go @@ -3,25 +3,56 @@ package handlers import ( "errors" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/duke-git/lancet/v2/slice" + "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/dirutil" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) // responsibleUserID returns the sender's ID. Callback queries carry it -// natively; message updates resolve it through the entity map. +// natively; message updates resolve the sender through the entity map, which +// also covers group and supergroup chats. func responsibleUserID(u *ext.Update) int64 { - if u.CallbackQuery != nil { - return u.CallbackQuery.GetUserID() + return tgutil.SenderID(u) +} + +// chatID returns the chat that replies and edits must target. In private chats +// this equals the sender ID; in groups and supergroups it is the group chat. +func chatID(u *ext.Update) int64 { + return tgutil.ChatID(u) +} + +// isGroupChat reports whether the update belongs to a basic group or a +// supergroup. Broadcast channels are not group chats. +func isGroupChat(u *ext.Update) bool { + if u == nil { + return false + } + if message := u.EffectiveMessage; message != nil { + switch message.PeerID.(type) { + case *tg.PeerUser: + return false + case *tg.PeerChat: + return true + case *tg.PeerChannel: + channel := u.GetChannel() + return channel == nil || channel.Megagroup + } + } + if u.GetChat() != nil { + return true } - return u.GetUserChat().GetID() + channel := u.GetChannel() + return channel != nil && channel.Megagroup } func checkPermission(ctx *ext.Context, update *ext.Update) error { @@ -29,7 +60,9 @@ func checkPermission(ctx *ext.Context, update *ext.Update) error { if !slice.Contain(config.C().GetUsersID(), userID) { if cbq := update.CallbackQuery; cbq != nil { ctx.AnswerCallback(msgelem.AlertCallbackAnswer(cbq.GetQueryID(), i18n.T(i18nk.BotMsgCommonErrorNoPermission, nil))) - } else { + } else if !isGroupChat(update) { + // Stay silent in groups: unaddressed traffic and unauthorized + // command attempts must not turn into reply spam. ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorNoPermission, nil)), nil) } return dispatcher.EndGroups @@ -51,9 +84,20 @@ func withPermission(handler func(*ext.Context, *ext.Update) error) func(*ext.Con } } +// skipLinkForMedia lets the media handler save an attached file instead of a +// link that appears in its caption. +func skipLinkForMedia(next func(*ext.Context, *ext.Update) error) func(*ext.Context, *ext.Update) error { + return func(ctx *ext.Context, update *ext.Update) error { + if message := update.EffectiveMessage; message != nil && message.Media != nil && mediautil.IsSupported(message.Media) { + return dispatcher.ContinueGroups + } + return next(ctx, update) + } +} + func handleSilentMode(next func(*ext.Context, *ext.Update) error, handler func(*ext.Context, *ext.Update) error) func(*ext.Context, *ext.Update) error { return func(ctx *ext.Context, update *ext.Update) error { - userID := update.GetUserChat().GetID() + userID := responsibleUserID(update) user, err := database.GetUserByChatID(ctx, userID) if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetUserInfoFailed, map[string]any{ @@ -64,28 +108,30 @@ func handleSilentMode(next func(*ext.Context, *ext.Update) error, handler func(* if !user.Silent { return next(ctx, update) } - if user.DefaultStorage == "" { + if user.DefaultStorageID == 0 { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorDefaultStorageNotSet, nil)), nil) return next(ctx, update) } - stor, err := storage.GetStorageByUserIDAndName(ctx, userID, user.DefaultStorage) + stor, err := storage.GetStorageByUserIDAndID(ctx, userID, user.DefaultStorageID) if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ "Error": err.Error(), })), nil) return dispatcher.EndGroups } - if user.DefaultDir != 0 { - dir, err := database.GetDirByID(ctx, user.DefaultDir) + base := ctx.Context + if user.DefaultDirID != 0 { + dir, err := database.GetDirByID(ctx, user.DefaultDirID) if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetDirFailed, map[string]any{ "Error": err.Error(), })), nil) return next(ctx, update) } - ctx.Context = dirutil.WithContext(ctx.Context, dir) + base = dirutil.WithContext(base, dir) } - ctx.Context = storage.WithContext(ctx.Context, stor) - return handler(ctx, update) + base = storage.WithContext(base, stor) + child := ext.NewContext(base, ctx.Raw, ctx.PeerStorage, ctx.Self, ctx.Sender, ctx.Entities, false) + return handler(child, update) } } diff --git a/client/bot/handlers/middleware_test.go b/client/bot/handlers/middleware_test.go index f752c7bc..09e740ab 100644 --- a/client/bot/handlers/middleware_test.go +++ b/client/bot/handlers/middleware_test.go @@ -1,14 +1,12 @@ package handlers import ( - "os" - "path/filepath" "testing" - "github.com/celestix/gotgproto/ext" - "github.com/celestix/gotgproto/types" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/mygotg/ext" + "github.com/krau/mygotg/types" ) // Regression: callback queries usually arrive as updateShort without entity @@ -62,13 +60,9 @@ func TestResponsibleUserID(t *testing.T) { // it as an error, so every permitted callback was swallowed before the real // handler ran. func TestWithPermissionInvokesHandler(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.toml") - if err := os.WriteFile(path, []byte("workers = 2\n\n[[users]]\nid = 42\n"), 0o644); err != nil { - t.Fatal(err) - } - if err := config.Init(t.Context(), path); err != nil { - t.Fatal(err) - } + cfg := config.Default(t.TempDir()) + cfg.Users = []config.UserConfig{{ID: 42, Enabled: true}} + config.Set(cfg) update := &ext.Update{CallbackQuery: &tg.UpdateBotCallbackQuery{UserID: 42}} called := false diff --git a/client/bot/handlers/parse.go b/client/bot/handlers/parse.go index 907140d8..fbc22816 100644 --- a/client/bot/handlers/parse.go +++ b/client/bot/handlers/parse.go @@ -7,8 +7,6 @@ import ( "path" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/dirutil" @@ -22,6 +20,8 @@ import ( "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleTextMessage(ctx *ext.Context, u *ext.Update) error { @@ -67,7 +67,7 @@ func handleTextMessage(ctx *ext.Context, u *ext.Update) error { return dispatcher.EndGroups } logger.Debug("Parsed item from text message", "title", item.Title, "url", item.URL) - userID := u.GetUserChat().GetID() + userID := responsibleUserID(u) markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, userID), tcbdata.Add{ TaskType: tasktype.TaskTypeParseditem, ParsedItem: item, @@ -87,7 +87,7 @@ func handleTextMessage(ctx *ext.Context, u *ext.Update) error { })), nil) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(u), &tg.MessagesEditMessageRequest{ Message: text, ReplyMarkup: markup, Entities: entities, @@ -116,7 +116,6 @@ func handleSilentSaveText(ctx *ext.Context, u *ext.Update) error { return dispatcher.EndGroups } logger.Debug("Parsed item from text message", "title", item.Title, "url", item.URL) - userID := u.GetUserChat().GetID() text, entities, err := msgelem.BuildParsedTextEntity(*item) if err != nil { logger.Errorf("Failed to build parsed text entity: %s", err) @@ -125,12 +124,12 @@ func handleSilentSaveText(ctx *ext.Context, u *ext.Update) error { })), nil) return dispatcher.EndGroups } - msg, err := ctx.SendMessage(userID, &tg.MessagesSendMessageRequest{ + msg, err := ctx.SendMessage(chatID(u), &tg.MessagesSendMessageRequest{ Message: text, Entities: entities, ReplyTo: &tg.InputReplyToMessage{ ReplyToMsgID: u.EffectiveMessage.ID, - ReplyToPeerID: u.GetUserChat().AsInputPeer(), + ReplyToPeerID: u.EffectiveChat().GetInputPeer(), }, }) if err != nil { @@ -144,5 +143,5 @@ func handleSilentSaveText(ctx *ext.Context, u *ext.Update) error { if p := dirutil.PathFromContext(ctx); p != "" { dirPath = path.Join(p, dirPath) } - return shortcut.CreateAndAddParsedTaskWithEdit(ctx, stor, dirPath, item, msg.ID, userID) + return shortcut.CreateAndAddParsedTaskWithEdit(ctx, stor, dirPath, item, msg.ID, chatID(u)) } diff --git a/client/bot/handlers/parser.go b/client/bot/handlers/parser.go index 651e781e..a228ae0a 100644 --- a/client/bot/handlers/parser.go +++ b/client/bot/handlers/parser.go @@ -4,13 +4,13 @@ import ( "bytes" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/parsers" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleParserCmd(ctx *ext.Context, u *ext.Update) error { diff --git a/client/bot/handlers/register.go b/client/bot/handlers/register.go index 624b8725..c577053f 100644 --- a/client/bot/handlers/register.go +++ b/client/bot/handlers/register.go @@ -3,16 +3,15 @@ package handlers import ( "regexp" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/dispatcher/handlers" - "github.com/celestix/gotgproto/dispatcher/handlers/filters" - "github.com/celestix/gotgproto/ext" + "github.com/gotd/td/tg" sabotfilters "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/filters" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/re" - userclient "github.com/krau/SaveAny-Bot/client/user" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" - "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/pkg/tcbdata" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/dispatcher/handlers" + "github.com/krau/mygotg/dispatcher/handlers/filters" + "github.com/krau/mygotg/ext" ) type DescCommandHandler struct { @@ -23,50 +22,70 @@ type DescCommandHandler struct { var CommandHandlers = []DescCommandHandler{ {"start", i18nk.BotMsgCmdStart, handleHelpCmd}, - {"silent", i18nk.BotMsgCmdSilent, handleSilentCmd}, - {"storage", i18nk.BotMsgCmdStorage, handleStorageCmd}, - {"dir", i18nk.BotMsgCmdDir, handleDirCmd}, - {"rule", i18nk.BotMsgCmdRule, handleRuleCmd}, {"save", i18nk.BotMsgCmdSave, handleSilentMode(handleSaveCmd, handleSilentSaveReplied)}, - {"dl", i18nk.BotMsgCmdDl, handleDlCmd}, - {"aria2dl", i18nk.BotMsgCmdAria2dl, handleAria2DlCmd}, - {"ytdlp", i18nk.BotMsgCmdYtdlp, handleYtdlpCmd}, + {"download", i18nk.BotMsgCmdDownload, handleDownloadCmd}, {"transfer", i18nk.BotMsgCmdTransfer, handleTransferCmd}, - {"task", i18nk.BotMsgCmdTask, handleTaskCmd}, - {"cancel", i18nk.BotMsgCmdCancel, handleCancelCmd}, + {"tasks", i18nk.BotMsgCmdTasks, handleTasksCmd}, {"config", i18nk.BotMsgCmdConfig, handleConfigCmd}, - {"fnametmpl", i18nk.BotMsgCmdFnametmpl, handleConfigFnameTmpl}, - {"help", i18nk.BotMsgCmdHelp, handleHelpCmd}, - {"parser", i18nk.BotMsgCmdParser, handleParserCmd}, + {"template", i18nk.BotMsgCmdTemplate, handleTemplateCmd}, + {"dirs", i18nk.BotMsgCmdDirs, handleDirCmd}, + {"rules", i18nk.BotMsgCmdRules, handleRuleCmd}, {"watch", i18nk.BotMsgCmdWatch, handleWatchCmd}, - {"unwatch", i18nk.BotMsgCmdUnwatch, handleUnwatchCmd}, - {"lswatch", i18nk.BotMsgCmdLswatch, handleLswatchCmd}, - {"syncpeers", i18nk.BotMsgCmdSyncpeers, handleSyncpeersCmd}, + {"parser", i18nk.BotMsgCmdParser, handleParserCmd}, {"update", i18nk.BotMsgCmdUpdate, handleUpdateCmd}, } -func Register(disp dispatcher.Dispatcher, botCtx *ext.Context) { - disp.AddHandler(handlers.NewMessage(filters.Message.ChatType(filters.ChatTypeChannel), func(ctx *ext.Context, u *ext.Update) error { +// groupGate admits private chats, basic groups, and supergroups. Broadcast +// channels and updates without a resolvable human sender are rejected. +func groupGate(_ *ext.Context, u *ext.Update) error { + message := u.EffectiveMessage + if message == nil || message.Out { return dispatcher.EndGroups - })) - disp.AddHandler(handlers.NewMessage(filters.Message.ChatType(filters.ChatTypeChat), func(ctx *ext.Context, u *ext.Update) error { + } + switch message.PeerID.(type) { + case *tg.PeerUser, *tg.PeerChat: + case *tg.PeerChannel: + // The channel entity can be absent from short updates. Broadcast + // channel posts are rejected by the sender check below because their + // sender is the channel, not a user. + default: return dispatcher.EndGroups - })) + } + if responsibleUserID(u) == 0 { + return dispatcher.EndGroups + } + return dispatcher.ContinueGroups +} + +func Register(disp dispatcher.Dispatcher) { + disp.AddHandler(handlers.NewMessage(filters.Message.All, groupGate)) disp.AddHandler(handlers.NewMessage(filters.Message.All, checkPermission)) for _, info := range CommandHandlers { disp.AddHandler(handlers.NewCommand(info.Cmd, info.handler)) } + // Group chats only react to explicit commands and callback buttons. Plain + // messages would otherwise save every album or link posted in the group + // once privacy mode is disabled. + disp.AddHandler(handlers.NewMessage(filters.Message.All, func(ctx *ext.Context, u *ext.Update) error { + if isGroupChat(u) { + return dispatcher.EndGroups + } + return dispatcher.ContinueGroups + })) disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix("update"), withPermission(handleUpdateCallback))) disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeAdd), withPermission(handleAddCallback))) disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeSetDefault), withPermission(handleSetDefaultCallback))) disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeCancel), withPermission(handleCancelCallback))) disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeConfig), withPermission(handleConfigCallback))) - disp.AddHandler(handlers.NewMessage(sabotfilters.RegexUrl(regexp.MustCompile(re.TgMessageLinkRegexString)), handleSilentMode(handleMessageLink, handleSilentSaveLink))) - disp.AddHandler(handlers.NewMessage(sabotfilters.RegexUrl(regexp.MustCompile(re.TelegraphUrlRegexString)), handleSilentMode(handleTelegraphUrlMessage, handleSilentSaveTelegraph))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeEngine), withPermission(handleEngineCallback))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeTask), withPermission(handleTaskCallback))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeRetry), withPermission(handleRetryCallback))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeWatch), withPermission(handleWatchCallback))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeDir), withPermission(handleDirCallback))) + disp.AddHandler(handlers.NewCallbackQuery(filters.CallbackQuery.Prefix(tcbdata.TypeRule), withPermission(handleRuleCallback))) + disp.AddHandler(handlers.NewMessage(sabotfilters.RegexUrl(regexp.MustCompile(re.TgMessageLinkRegexString)), skipLinkForMedia(handleSilentMode(handleMessageLink, handleSilentSaveLink)))) + disp.AddHandler(handlers.NewMessage(sabotfilters.RegexUrl(regexp.MustCompile(re.TelegraphUrlRegexString)), skipLinkForMedia(handleSilentMode(handleTelegraphUrlMessage, handleSilentSaveTelegraph)))) disp.AddHandler(handlers.NewMessage(filters.Message.Media, handleSilentMode(handleMediaMessage, handleSilentSaveMedia))) disp.AddHandler(handlers.NewMessage(filters.Message.Text, handleSilentMode(handleTextMessage, handleSilentSaveText))) - if config.C().Telegram.Userbot.Enable { - go listenMediaMessageEvent(userclient.GetMediaMessageCh(), botCtx) - } } diff --git a/client/bot/handlers/rule.go b/client/bot/handlers/rule.go index 7d7bf11e..3179c941 100644 --- a/client/bot/handlers/rule.go +++ b/client/bot/handlers/rule.go @@ -5,10 +5,10 @@ import ( "strconv" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/duke-git/lancet/v2/slice" + "github.com/gotd/td/telegram/message/markup" + "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" @@ -16,12 +16,33 @@ import ( "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/rule" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) +// resolveRuleStorage accepts a storage ID or a storage name. IDs avoid the +// ambiguity of names that contain spaces; CHOSEN keeps the rule storage-less. +func resolveRuleStorage(userChatID int64, arg string) (uint, error) { + if strings.EqualFold(arg, rule.RuleStorNameChosen) { + return 0, nil + } + if id, err := strconv.ParseUint(arg, 10, 32); err == nil { + if !config.C().HasStorageID(userChatID, uint(id)) { + return 0, fmt.Errorf("storage ID %d not found", id) + } + return uint(id), nil + } + if !config.C().HasStorage(userChatID, arg) { + return 0, fmt.Errorf("storage not found: %s", arg) + } + return config.C().GetStorageIDByName(arg), nil +} + func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { logger := log.FromContext(ctx) args := strutil.ParseArgsRespectQuotes(update.EffectiveMessage.Text) - userChatID := update.GetUserChat().GetID() + userChatID := responsibleUserID(update) user, err := database.GetUserByChatID(ctx, userChatID) if err != nil { logger.Errorf("Failed to get user rules: %s", err) @@ -29,12 +50,12 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildRuleHelpStyling(user.ApplyRule, user.Rules)), nil) + ctx.Reply(update, ext.ReplyTextString(ruleMenuText(user)), &ext.ReplyOpts{Markup: ruleMenuMarkup(user)}) return dispatcher.EndGroups } switch args[1] { case "switch": - // /rule switch + // /rules switch applyRule := !user.ApplyRule if err := database.UpdateUserApplyRule(ctx, user.ChatID, applyRule); err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorUpdateUserFailed, nil)), nil) @@ -46,7 +67,7 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleInfoRuleModeDisabled, nil)), nil) } case "add": - // /rule add + // /rules add if len(args) < 6 { ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildRuleHelpStyling(user.ApplyRule, user.Rules)), nil) return dispatcher.EndGroups @@ -69,15 +90,22 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { } ruleData := args[3] - storageName := args[4] + storageArg := args[4] dirPath := args[5] + storageID, err := resolveRuleStorage(user.ChatID, storageArg) + if err != nil { + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorStorageNotFound, map[string]any{ + "Storage": storageArg, + })), nil) + return dispatcher.EndGroups + } rd := &database.Rule{ - Type: ruleType.String(), - Data: ruleData, - StorageName: storageName, - DirPath: dirPath, - UserID: user.ID, + Type: ruleType.String(), + Content: ruleData, + StorageID: storageID, + DirPath: dirPath, + UserID: user.ID, } if err := database.CreateRule(ctx, rd); err != nil { logger.Errorf("failed to create rule: %s", err) @@ -86,15 +114,16 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { } ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleInfoCreateRuleSuccess, nil)), nil) case "preset": - // /rule preset [base_path] + // /rules preset [base_path] if len(args) < 3 { ctx.Reply(update, ext.ReplyTextStyledTextArray(msgelem.BuildRuleHelpStyling(user.ApplyRule, user.Rules)), nil) return dispatcher.EndGroups } - storageName := args[2] - if !config.C().HasStorage(user.ChatID, storageName) { + storageArg := args[2] + storageID, err := resolveRuleStorage(user.ChatID, storageArg) + if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorStorageNotFound, map[string]any{ - "Storage": storageName, + "Storage": storageArg, })), nil) return dispatcher.EndGroups } @@ -106,11 +135,11 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { imported := 0 for _, p := range presets { rd := &database.Rule{ - Type: rule.FileNameRegex.String(), - Data: p.Regex, - StorageName: storageName, - DirPath: p.Dir, - UserID: user.ID, + Type: rule.FileNameRegex.String(), + Content: p.Regex, + StorageID: storageID, + DirPath: p.Dir, + UserID: user.ID, } if err := database.CreateRule(ctx, rd); err != nil { logger.Errorf("failed to create preset rule %s: %s", p.Name, err) @@ -126,7 +155,7 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { "Count": imported, })), nil) case "del": - // /rule del + // /rules del if len(args) < 3 { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRulePromptProvideRuleId, nil)), nil) return dispatcher.EndGroups @@ -137,7 +166,7 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorInvalidRuleId, nil)), nil) return dispatcher.EndGroups } - if err := database.DeleteRule(ctx, uint(id)); err != nil { + if err := database.DeleteRule(ctx, uint(id), user.ID); err != nil { logger.Errorf("failed to delete rule %d: %s", id, err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgRuleErrorDeleteRuleFailed, nil)), nil) return dispatcher.EndGroups @@ -149,3 +178,88 @@ func handleRuleCmd(ctx *ext.Context, update *ext.Update) error { } return dispatcher.EndGroups } + +func ruleMenuText(user *database.User) string { + var b strings.Builder + b.WriteString(i18n.T(i18nk.BotMsgRuleMenuTitle)) + b.WriteString("\n\n") + b.WriteString(i18n.T(i18nk.BotMsgRuleMenuHint)) + mode := i18nk.BotMsgRuleInfoRuleModeDisabled + if user.ApplyRule { + mode = i18nk.BotMsgRuleInfoRuleModeEnabled + } + b.WriteString("\n\n") + b.WriteString(i18n.T(mode)) + if len(user.Rules) == 0 { + b.WriteString("\n\n") + b.WriteString(i18n.T(i18nk.BotMsgRuleMenuEmpty)) + return b.String() + } + for _, r := range user.Rules { + storageName := rule.RuleStorNameChosen + if r.StorageID != 0 { + storageName = config.C().GetStorageNameByID(r.StorageID) + if storageName == "" { + storageName = fmt.Sprintf("#%d", r.StorageID) + } + } + fmt.Fprintf(&b, "\n\n#%d %s %s %s %s", r.ID, r.Type, r.Content, storageName, r.DirPath) + } + return b.String() +} + +func ruleMenuMarkup(user *database.User) *tg.ReplyInlineMarkup { + rows := make([]tg.KeyboardInlineButtonRow, 0, len(user.Rules)+1) + label := i18nk.BotMsgRuleButtonEnable + if user.ApplyRule { + label = i18nk.BotMsgRuleButtonDisable + } + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(label), fmt.Appendf(nil, "%s switch", tcbdata.TypeRule)), + }}) + for _, rule := range user.Rules { + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgRuleButtonDelete), fmt.Appendf(nil, "%s del %d", tcbdata.TypeRule, rule.ID)), + }}) + } + return &tg.ReplyInlineMarkup{Rows: rows} +} + +func handleRuleCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + switch args[1] { + case "switch": + if err := database.UpdateUserApplyRule(ctx, user.ChatID, !user.ApplyRule); err != nil { + return err + } + case "del": + if len(args) < 3 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } + ruleID, err := strconv.Atoi(args[2]) + if err != nil { + return err + } + if err := database.DeleteRule(ctx, uint(ruleID), user.ID); err != nil { + return err + } + } + user, err = database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: ruleMenuText(user), + ReplyMarkup: ruleMenuMarkup(user), + }) + return dispatcher.EndGroups +} diff --git a/client/bot/handlers/save.go b/client/bot/handlers/save.go index 4500b910..79bd3be5 100644 --- a/client/bot/handlers/save.go +++ b/client/bot/handlers/save.go @@ -4,8 +4,6 @@ import ( "regexp" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/duke-git/lancet/v2/validator" "github.com/gotd/td/tg" @@ -16,11 +14,14 @@ import ( "github.com/krau/SaveAny-Bot/client/user" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/mediafilter" "github.com/krau/SaveAny-Bot/common/utils/strutil" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/krau/SaveAny-Bot/storage" ) @@ -36,7 +37,7 @@ func handleSaveCmd(ctx *ext.Context, update *ext.Update) error { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgSaveHelpText)), nil) return dispatcher.EndGroups } - userDB, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID()) + userDB, err := database.GetUserByChatID(ctx, responsibleUserID(update)) if err != nil { return err } @@ -48,7 +49,7 @@ func handleSaveCmd(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - userId := update.GetUserChat().GetID() + userId := responsibleUserID(update) stors := storage.GetUserStorages(ctx, userId) req, err := msgelem.BuildAddOneSelectStorageMessage(ctx, stors, file, msg.ID) if err != nil { @@ -71,7 +72,7 @@ func handleSilentSaveReplied(ctx *ext.Context, update *ext.Update) error { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgSaveHelpText)), nil) return dispatcher.EndGroups } - userDB, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID()) + userDB, err := database.GetUserByChatID(ctx, responsibleUserID(update)) if err != nil { return err } @@ -83,7 +84,7 @@ func handleSilentSaveReplied(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, update.GetUserChat().GetID(), stor, dirutil.PathFromContext(ctx), file, msg.GetID()) + return shortcut.CreateAndAddTGFileTaskWithEdit(ctx, responsibleUserID(update), chatID(update), stor, dirutil.PathFromContext(ctx), file, msg.GetID()) } func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error { @@ -110,7 +111,7 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error if uctx != nil && validator.IsIntStr(chatArg) { tctx = uctx } - chatID, err := tgutil.ParseChatID(tctx, chatArg) + sourceChatID, err := tgutil.ParseChatID(tctx, chatArg) if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorInvalidIdOrUsername, map[string]any{"Error": err.Error()})), nil) return dispatcher.EndGroups @@ -122,7 +123,7 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error return dispatcher.EndGroups } - msgs, err := tgutil.GetMessagesRange(tctx, chatID, int(startID), int(endID)) + msgs, err := tgutil.GetMessagesRange(tctx, sourceChatID, int(startID), int(endID)) if err != nil { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetMessagesFailed, map[string]any{"Error": err.Error()})), nil) return dispatcher.EndGroups @@ -141,8 +142,7 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error if !ok { continue } - supported := mediautil.IsSupported(media) - if !supported { + if !mediautil.IsSupported(media) || !mediafilter.Allowed(media) { continue } file, err := tfile.FromMediaMessage(media, tctx.Raw, msg, tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*msg))) @@ -154,8 +154,12 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error sb.Reset() sb.WriteString(msg.GetMessage()) sb.WriteString(" ") - fn, _ := tgutil.GetMediaFileName(media) - sb.WriteString(fn) + fn, err := tgutil.GetMediaFileName(media) + if err != nil { + log.FromContext(ctx).Debugf("Failed to derive media filename for filter: %v", err) + } else { + sb.WriteString(fn) + } if !filter.MatchString(sb.String()) { continue } @@ -169,7 +173,7 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error stor := storage.FromContext(ctx) if stor == nil { // not in silent mode - stors := storage.GetUserStorages(ctx, update.GetUserChat().GetID()) + stors := storage.GetUserStorages(ctx, responsibleUserID(update)) markup, err := msgelem.BuildAddSelectStorageKeyboard(stors, tcbdata.Add{ Files: files, }) @@ -188,5 +192,5 @@ func handleBatchSave(ctx *ext.Context, update *ext.Update, args []string) error }) return dispatcher.EndGroups } - return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, update.GetUserChat().GetID(), stor, "", files, replied.ID) + return shortcut.CreateAndAddBatchTGFileTaskWithEdit(ctx, responsibleUserID(update), chatID(update), stor, "", files, replied.ID) } diff --git a/client/bot/handlers/silent.go b/client/bot/handlers/silent.go index f83c5fcb..eec2cd0e 100644 --- a/client/bot/handlers/silent.go +++ b/client/bot/handlers/silent.go @@ -4,8 +4,6 @@ import ( "fmt" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/cache" @@ -14,35 +12,10 @@ import ( "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) -func handleSilentCmd(ctx *ext.Context, update *ext.Update) error { - user, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID()) - if err != nil { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetUserInfoFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return nil - } - if !user.Silent && user.DefaultStorage == "" { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorDefaultStorageNotSet, nil)), nil) - return nil - } - user.Silent = !user.Silent - if err := database.UpdateUser(ctx, user); err != nil { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorUpdateUserInfoFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return nil - } - if user.Silent { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoSilentModeOn, nil)), nil) - } else { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoSilentModeOff, nil)), nil) - } - return dispatcher.EndGroups -} - func handleSetDefaultCallback(ctx *ext.Context, update *ext.Update) error { dataParts := strings.Split(string(update.CallbackQuery.Data), " ") if len(dataParts) < 2 { @@ -66,8 +39,8 @@ func handleSetDefaultCallback(ctx *ext.Context, update *ext.Update) error { } userID := update.CallbackQuery.GetUserID() - storageName := data.StorageName - selectedStorage, err := storage.GetStorageByUserIDAndName(ctx, userID, storageName) + storageID := data.StorageID + selectedStorage, err := storage.GetStorageByUserIDAndID(ctx, userID, storageID) if err != nil { return failedAnswer(i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ "Error": err.Error(), @@ -86,22 +59,22 @@ func handleSetDefaultCallback(ctx *ext.Context, update *ext.Update) error { if err != nil { return failedAnswer(i18n.T(i18nk.BotMsgDirErrorGetUserDirsFailed, nil)) } - user.DefaultDir = dir.ID + user.DefaultDirID = dir.ID } else { - dirs, err := database.GetDirsByUserIDAndStorageName(ctx, user.ID, storageName) + dirs, err := database.GetDirsByUserIDAndStorageID(ctx, user.ID, storageID) if err != nil { return failedAnswer(i18n.T(i18nk.BotMsgCommonErrorGetDirFailed, map[string]any{ "Error": err.Error(), })) } if len(dirs) > 0 { - markup, err := msgelem.BuildSetDefaultDirMarkup(ctx, storageName, dirs) + markup, err := msgelem.BuildSetDefaultDirMarkup(ctx, storageID, dirs) if err != nil { return failedAnswer(i18n.T(i18nk.BotMsgCommonErrorBuildDirSelectKeyboardFailed, map[string]any{ "Error": err.Error(), })) } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgCommonPromptSelectDefaultDir, nil), ReplyMarkup: markup, @@ -109,7 +82,7 @@ func handleSetDefaultCallback(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } } - user.DefaultStorage = selectedStorage.Name() + user.DefaultStorageID = storageID if err := database.UpdateUser(ctx, user); err != nil { return failedAnswer(i18n.T(i18nk.BotMsgCommonErrorUpdateUserInfoFailed, map[string]any{ "Error": err.Error(), @@ -124,29 +97,9 @@ func handleSetDefaultCallback(ctx *ext.Context, update *ext.Update) error { "Dir": strings.TrimPrefix(dir.Path, "/"), }) } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ ID: update.CallbackQuery.GetMsgID(), Message: msg, }) return dispatcher.EndGroups } - -func handleStorageCmd(ctx *ext.Context, update *ext.Update) error { - userID := update.GetUserChat().GetID() - storages := storage.GetUserStorages(ctx, userID) - if len(storages) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorNoAvailableStorage, nil)), nil) - return nil - } - markup, err := msgelem.BuildSetDefaultStorageMarkup(ctx, storages) - if err != nil { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return nil - } - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonPromptSelectDefaultStorage, nil)), &ext.ReplyOpts{ - Markup: markup, - }) - return dispatcher.EndGroups -} diff --git a/client/bot/handlers/sync_peers.go b/client/bot/handlers/sync_peers.go index c3f0057f..7cc4b8b8 100644 --- a/client/bot/handlers/sync_peers.go +++ b/client/bot/handlers/sync_peers.go @@ -1,62 +1,10 @@ package handlers import ( - "context" - "sync" - - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" - "github.com/celestix/gotgproto/storage" - "github.com/charmbracelet/log" - "github.com/gotd/td/telegram/query/dialogs" - "github.com/krau/SaveAny-Bot/client/user" - "github.com/krau/SaveAny-Bot/common/i18n" - "github.com/krau/SaveAny-Bot/common/i18n/i18nk" - "github.com/krau/SaveAny-Bot/config" + userclient "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/mygotg/ext" ) -var syncpeerMu sync.Mutex - -func handleSyncpeersCmd(ctx *ext.Context, u *ext.Update) error { - if !config.C().Telegram.Userbot.Enable { - return dispatcher.EndGroups - } - syncpeerMu.Lock() - defer syncpeerMu.Unlock() - uctx := user.GetCtx() - if uctx == nil { - return dispatcher.EndGroups - } - ctx.Reply(u, ext.ReplyTextString(i18n.T(i18nk.BotMsgSyncpeersStart)), nil) - tapi := uctx.Raw - peerStorage := uctx.PeerStorage - log.FromContext(ctx).Info("Starting to sync peers...") - count := 0 - err := dialogs.NewQueryBuilder(tapi).GetDialogs().BatchSize(50).ForEach(ctx, func(ctx context.Context, e dialogs.Elem) error { - for cid, channel := range e.Entities.Channels() { - peerStorage.AddPeer(cid, channel.AccessHash, storage.TypeChannel, channel.Username) - count++ - } - for uid, user := range e.Entities.Users() { - peerStorage.AddPeer(uid, user.AccessHash, storage.TypeUser, user.Username) - count++ - } - for gid := range e.Entities.Chats() { - peerStorage.AddPeer(gid, storage.DefaultAccessHash, storage.TypeChat, storage.DefaultUsername) - count++ - } - return nil - }) - if err != nil { - log.FromContext(ctx).Error("Failed to sync peers", "error", err) - ctx.Reply(u, ext.ReplyTextString(i18n.T(i18nk.BotMsgSyncpeersFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return dispatcher.EndGroups - } - log.FromContext(ctx).Info("Finished syncing peers") - ctx.Reply(u, ext.ReplyTextString(i18n.T(i18nk.BotMsgSyncpeersSuccess, map[string]any{ - "Count": count, - })), nil) - return dispatcher.EndGroups +func syncPeers(ctx *ext.Context) (int, error) { + return userclient.SyncPeers(ctx) } diff --git a/client/bot/handlers/tasks.go b/client/bot/handlers/tasks.go index ac1c3c07..1088223e 100644 --- a/client/bot/handlers/tasks.go +++ b/client/bot/handlers/tasks.go @@ -1,117 +1,229 @@ package handlers import ( + "fmt" "strings" "time" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" - "github.com/gotd/td/telegram/message/styling" + "github.com/gotd/td/telegram/message/markup" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" + userclient "github.com/krau/SaveAny-Bot/client/user" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/core/retry" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/queue" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) -func handleTaskCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - args := strings.Fields(update.EffectiveMessage.Text) - if len(args) == 1 { - showRunningTasks(ctx, update) - return dispatcher.EndGroups - } +const taskListLimit = 10 + +func handleTasksCmd(ctx *ext.Context, update *ext.Update) error { + ctx.Reply(update, ext.ReplyTextString(taskMenuText()), &ext.ReplyOpts{ + Markup: taskMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func taskMenuText() string { + return i18n.T(i18nk.BotMsgTasksMenuTitle) + "\n\n" + i18n.T(i18nk.BotMsgTasksMenuHint) +} + +func taskMenuMarkup() *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{ + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgTasksMenuRunning), fmt.Appendf(nil, "%s running", tcbdata.TypeTask)), + markup.Callback(i18n.T(i18nk.BotMsgTasksMenuQueued), fmt.Appendf(nil, "%s queued", tcbdata.TypeTask)), + }}, + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgTasksMenuFailed), fmt.Appendf(nil, "%s failed", tcbdata.TypeTask)), + }}, + }} +} +func taskBackMarkup() *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{{ + Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgTasksMenuBack), fmt.Appendf(nil, "%s menu", tcbdata.TypeTask)), + }, + }}} +} + +func handleTaskCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } switch args[1] { - case "running", "run", "r": - showRunningTasks(ctx, update) - case "queued", "queue", "q", "waiting": - showQueuedTasks(ctx, update) - case "cancel", "c": - if len(args) < 3 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTasksUsageCancel)), nil) - return dispatcher.EndGroups - } - taskID := args[2] - if err := core.CancelTask(ctx, taskID); err != nil { - logger.Errorf("Failed to cancel task %s: %v", taskID, err) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTasksCancelFailed, map[string]any{"Error": err.Error()})), nil) - return dispatcher.EndGroups - } - ctx.Reply(update, ext.ReplyTextStyledTextArray([]styling.StyledTextOption{ - styling.Plain(i18n.T(i18nk.BotMsgTasksCancelRequestedPrefix)), - styling.Code(taskID), - }), nil) + case "menu": + return editTaskMenu(ctx, update) + case "running": + return editRunningTasks(ctx, update) + case "queued": + return editQueuedTasks(ctx, update) + case "failed": + return editFailedTasks(ctx, update) default: - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTasksUsage)), nil) + return editTaskMenu(ctx, update) } +} + +func editTaskMenu(ctx *ext.Context, update *ext.Update) error { + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: taskMenuText(), + ReplyMarkup: taskMenuMarkup(), + }) return dispatcher.EndGroups } -func showRunningTasks(ctx *ext.Context, update *ext.Update) { - tasks := core.GetRunningTasks(ctx) - if len(tasks) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTasksRunningEmpty)), nil) - return +type taskListItem struct { + ID string + Title string + Created time.Time +} + +func taskItems(tasks []queue.TaskInfo) []taskListItem { + items := make([]taskListItem, 0, len(tasks)) + for _, task := range tasks { + items = append(items, taskListItem{ID: task.ID, Title: task.Title, Created: task.Created}) } - opts := make([]styling.StyledTextOption, 0, 2+len(tasks)*4) - opts = append(opts, - styling.Bold(i18n.T(i18nk.BotMsgTasksRunningTitle)), - styling.Plain(i18n.T(i18nk.BotMsgTasksTotalPrefix, map[string]any{"Count": len(tasks)})), - ) - for _, t := range tasks { - created := t.Created.In(time.Local).Format("2006-01-02 15:04:05") - status := i18n.T(i18nk.BotMsgTasksStatusRunning) - if t.Cancelled { - status = i18n.T(i18nk.BotMsgTasksStatusCancelRequested) - } - opts = append(opts, - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldId)), - styling.Code(t.ID), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldTitle)), - styling.Code(t.Title), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldCreated)), - styling.Code(created), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldStatus)), - styling.Code(status), + return items +} + +func taskListText(title string, total int, items []taskListItem) string { + var b strings.Builder + fmt.Fprintf(&b, "%s (%d)", title, total) + limit := min(len(items), taskListLimit) + for i := 0; i < limit; i++ { + item := items[i] + fmt.Fprintf(&b, "\n\n%s: %s\n%s: %s\n%s: %s", + i18n.T(i18nk.BotMsgTasksFieldId), item.ID, + i18n.T(i18nk.BotMsgTasksFieldTitle), item.Title, + i18n.T(i18nk.BotMsgTasksFieldCreated), item.Created.In(time.Local).Format("2006-01-02 15:04:05"), ) } - ctx.Reply(update, ext.ReplyTextStyledTextArray(opts), nil) + if len(items) > limit { + fmt.Fprintf(&b, "\n\n%s", i18n.T(i18nk.BotMsgTasksTruncatedNote, map[string]any{"Count": len(items)})) + } + return b.String() } -func showQueuedTasks(ctx *ext.Context, update *ext.Update) { +func cancelTaskRows(items []taskListItem) []tg.KeyboardInlineButtonRow { + rows := make([]tg.KeyboardInlineButtonRow, 0, min(len(items), taskListLimit)+1) + for i := 0; i < min(len(items), taskListLimit); i++ { + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgTasksCancelButton), fmt.Appendf(nil, "%s %s", tcbdata.TypeCancel, items[i].ID)), + }}) + } + return rows +} + +func editRunningTasks(ctx *ext.Context, update *ext.Update) error { + tasks := core.GetRunningTasks(ctx) + var text string + markup := taskBackMarkup() + if len(tasks) == 0 { + text = i18n.T(i18nk.BotMsgTasksRunningEmpty) + } else { + items := taskItems(tasks) + text = taskListText(i18n.T(i18nk.BotMsgTasksRunningTitle), len(items), items) + markup.Rows = append(cancelTaskRows(items), markup.Rows...) + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: text, + ReplyMarkup: markup, + }) + return dispatcher.EndGroups +} + +func editQueuedTasks(ctx *ext.Context, update *ext.Update) error { tasks := core.GetQueuedTasks(ctx) + var text string + markup := taskBackMarkup() if len(tasks) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTasksQueuedEmpty)), nil) - return + text = i18n.T(i18nk.BotMsgTasksQueuedEmpty) + } else { + items := taskItems(tasks) + text = taskListText(i18n.T(i18nk.BotMsgTasksQueuedTitle), len(items), items) + markup.Rows = append(cancelTaskRows(items), markup.Rows...) } - opts := make([]styling.StyledTextOption, 0, 2+len(tasks)*3) - opts = append(opts, - styling.Bold(i18n.T(i18nk.BotMsgTasksQueuedTitle)), - styling.Plain(i18n.T(i18nk.BotMsgTasksTotalPrefix, map[string]any{"Count": len(tasks)})), - ) - const maxShown = 10 - for i, t := range tasks { - if i >= maxShown { - break - } - created := t.Created.In(time.Local).Format("2006-01-02 15:04:05") - status := i18n.T(i18nk.BotMsgTasksStatusQueued) - if t.Cancelled { - status = i18n.T(i18nk.BotMsgTasksStatusCancelRequested) + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: text, + ReplyMarkup: markup, + }) + return dispatcher.EndGroups +} + +func editFailedTasks(ctx *ext.Context, update *ext.Update) error { + failed, _, err := database.ListTaskRecords(ctx, database.TaskFilter{Status: "failed", Limit: taskListLimit}) + if err != nil { + return err + } + partial, _, err := database.ListTaskRecords(ctx, database.TaskFilter{Status: "partial", Limit: taskListLimit}) + if err != nil { + return err + } + records := append(failed, partial...) + if len(records) > taskListLimit { + records = records[:taskListLimit] + } + var text string + rows := make([]tg.KeyboardInlineButtonRow, 0, len(records)+1) + if len(records) == 0 { + text = i18n.T(i18nk.BotMsgTasksFailedEmpty) + } else { + var b strings.Builder + fmt.Fprintf(&b, "%s (%d)", i18n.T(i18nk.BotMsgTasksFailedTitle), len(records)) + for _, record := range records { + fmt.Fprintf(&b, "\n\n%s: %s\n%s: %s\n%s: %s", + i18n.T(i18nk.BotMsgTasksFieldId), record.TaskID, + i18n.T(i18nk.BotMsgTasksFieldTitle), record.Title, + i18n.T(i18nk.BotMsgTasksFieldStatus), record.Status, + ) + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgTasksRetryButton), fmt.Appendf(nil, "%s %s", tcbdata.TypeRetry, record.TaskID)), + }}) } - opts = append(opts, - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldId)), - styling.Code(t.ID), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldTitle)), - styling.Code(t.Title), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldCreated)), - styling.Code(created), - styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksFieldStatus)), - styling.Code(status), - ) + text = b.String() + } + rows = append(rows, taskBackMarkup().Rows...) + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: text, + ReplyMarkup: &tg.ReplyInlineMarkup{Rows: rows}, + }) + return dispatcher.EndGroups +} + +func handleRetryCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) } - if len(tasks) > maxShown { - opts = append(opts, styling.Plain("\n"+i18n.T(i18nk.BotMsgTasksTruncatedNote, map[string]any{"Count": len(tasks)}))) + taskID := args[1] + candidates := []*ext.Context{ctx} + if uc := userclient.GetCtx(); uc != nil { + candidates = append(candidates, uc) } - ctx.Reply(update, ext.ReplyTextStyledTextArray(opts), nil) + newID, count, err := retry.FailedTask(ctx, candidates, taskID) + if err != nil { + log.FromContext(ctx).Errorf("failed to retry task %s: %v", taskID, err) + ctx.AnswerCallback(msgelem.AlertCallbackAnswer(update.CallbackQuery.GetQueryID(), i18n.T(i18nk.BotMsgRetryErrorFailed, map[string]any{ + "Error": err.Error(), + }))) + return dispatcher.EndGroups + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: i18n.T(i18nk.BotMsgRetryInfoStarted, map[string]any{"TaskID": newID, "Count": count}), + }) + return dispatcher.EndGroups } diff --git a/client/bot/handlers/telegraph.go b/client/bot/handlers/telegraph.go index 6f30600a..2e7397c6 100644 --- a/client/bot/handlers/telegraph.go +++ b/client/bot/handlers/telegraph.go @@ -4,8 +4,6 @@ import ( "fmt" "path" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/telegram/message/entity" "github.com/gotd/td/telegram/message/styling" @@ -18,6 +16,8 @@ import ( "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) func handleTelegraphUrlMessage(ctx *ext.Context, update *ext.Update) error { @@ -27,7 +27,7 @@ func handleTelegraphUrlMessage(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - userID := update.GetUserChat().GetID() + userID := responsibleUserID(update) stors := storage.GetUserStorages(ctx, userID) markup, err := msgelem.BuildAddSelectStorageKeyboard(stors, tcbdata.Add{ TaskType: tasktype.TaskTypeTphpics, @@ -55,7 +55,7 @@ func handleTelegraphUrlMessage(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } text, entities := eb.Complete() - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ Message: text, ID: msg.ID, ReplyMarkup: markup, @@ -70,11 +70,10 @@ func handleSilentSaveTelegraph(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - userID := update.GetUserChat().GetID() dirpath := result.TphDir if p := dirutil.PathFromContext(ctx); p != "" { dirpath = path.Join(p, dirpath) } - return shortcut.CreateAndAddtelegraphWithEdit(ctx, userID, result.Page, dirpath, result.Pics, stor, msg.ID) + return shortcut.CreateAndAddtelegraphWithEdit(ctx, chatID(update), result.Page, dirpath, result.Pics, stor, msg.ID) } diff --git a/client/bot/handlers/transfer.go b/client/bot/handlers/transfer.go index f0d0a6e8..0d148039 100644 --- a/client/bot/handlers/transfer.go +++ b/client/bot/handlers/transfer.go @@ -3,10 +3,9 @@ package handlers import ( "fmt" "regexp" + "sort" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" @@ -14,15 +13,36 @@ import ( "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/common/utils/strutil" "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/core/tasks/transfer" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/storagetypes" "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) +// splitTransferSource splits "storage:/path" while allowing storage names +// that contain spaces or colons. Known storage names are matched longest-first. +func splitTransferSource(userID int64, raw string) (string, string, bool) { + names := config.C().GetStorageNamesByUserID(userID) + sort.SliceStable(names, func(i, j int) bool { return len(names[i]) > len(names[j]) }) + for _, name := range names { + prefix := name + ":" + if after, ok := strings.CutPrefix(raw, prefix); ok { + return name, after, true + } + } + parts := strings.SplitN(raw, ":", 2) + if len(parts) != 2 || parts[0] == "" { + return "", "", false + } + return parts[0], parts[1], true +} + func handleTransferCmd(ctx *ext.Context, update *ext.Update) error { logger := log.FromContext(ctx) args := strutil.ParseArgsRespectQuotes(update.EffectiveMessage.Text) @@ -32,16 +52,12 @@ func handleTransferCmd(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } - // Parse source: storage_name:/path - sourceParts := strings.SplitN(args[1], ":", 2) - if len(sourceParts) != 2 { + userID := responsibleUserID(update) + sourceStorageName, sourcePath, ok := splitTransferSource(userID, args[1]) + if !ok { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgTransferErrorInvalidSource, nil)), nil) return dispatcher.EndGroups } - sourceStorageName := sourceParts[0] - sourcePath := sourceParts[1] - - userID := update.GetUserChat().GetID() sourceStorage, err := storage.GetStorageByUserIDAndName(ctx, userID, sourceStorageName) if err != nil { @@ -123,10 +139,10 @@ func handleTransferCmd(ctx *ext.Context, update *ext.Update) error { } markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, userID), tcbdata.Add{ - TaskType: tasktype.TaskTypeTransfer, - TransferSourceStorName: sourceStorageName, - TransferSourcePath: sourcePath, - TransferFiles: filePaths, + TaskType: tasktype.TaskTypeTransfer, + TransferSourceStorageID: storage.ID(sourceStorage), + TransferSourcePath: sourcePath, + TransferFiles: filePaths, }) if err != nil { logger.Errorf("Failed to build storage selection keyboard: %s", err) @@ -149,38 +165,38 @@ func handleTransferCmd(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } -func handleTransferCallback(ctx *ext.Context, userID int64, targetStorage storage.Storage, dirPath string, data tcbdata.Add, msgID int) error { +func handleTransferCallback(ctx *ext.Context, userID, chatID int64, targetStorage storage.Storage, dirPath string, data tcbdata.Add, msgID int) error { logger := log.FromContext(ctx) - sourceStorage, err := storage.GetStorageByUserIDAndName(ctx, userID, data.TransferSourceStorName) + sourceStorage, err := storage.GetStorageByUserIDAndID(ctx, userID, data.TransferSourceStorageID) if err != nil { logger.Errorf("Failed to get source storage: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, - Message: i18n.T(i18nk.BotMsgTransferErrorStorageNotFound, map[string]any{"StorageName": data.TransferSourceStorName, "Error": err}), + Message: i18n.T(i18nk.BotMsgTransferErrorStorageNotFound, map[string]any{"StorageName": config.C().GetStorageNameByID(data.TransferSourceStorageID), "Error": err}), }) return dispatcher.EndGroups } listable, ok := sourceStorage.(storage.StorageListable) if !ok { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, - Message: i18n.T(i18nk.BotMsgTransferErrorStorageNotListable, map[string]any{"StorageName": data.TransferSourceStorName}), + Message: i18n.T(i18nk.BotMsgTransferErrorStorageNotListable, map[string]any{"StorageName": config.C().GetStorageNameByID(data.TransferSourceStorageID)}), }) return dispatcher.EndGroups } // Re-fetch files to get FileInfo (since we only stored paths) // This is necessary to get size and other metadata - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgTransferInfoFetchingFiles, nil), }) allFiles, err := listable.ListFiles(ctx, data.TransferSourcePath) if err != nil { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgTransferErrorListFilesFailed, map[string]any{"Error": err}), }) @@ -206,7 +222,7 @@ func handleTransferCallback(ctx *ext.Context, userID int64, targetStorage storag } if len(elems) == 0 { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgTransferErrorNoFilesToTransfer, nil), }) @@ -219,19 +235,19 @@ func handleTransferCallback(ctx *ext.Context, userID int64, targetStorage storag taskID, injectCtx, elems, - transfer.NewProgressTracker(msgID, userID), + transfer.NewProgressTracker(msgID, chatID), true, // IgnoreErrors ) if err := core.AddTask(injectCtx, task); err != nil { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgTransferErrorAddTaskFailed, map[string]any{"Error": err}), }) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgTransferInfoTaskAdded, map[string]any{ "Count": len(elems), diff --git a/client/bot/handlers/update.go b/client/bot/handlers/update.go index 30b3515e..50d3895e 100644 --- a/client/bot/handlers/update.go +++ b/client/bot/handlers/update.go @@ -1,20 +1,20 @@ package handlers import ( - "errors" "regexp" "strings" "github.com/blang/semver" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/gotd/td/telegram/message/html" + "github.com/gotd/td/telegram/message/markup" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/common/selfupdate" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/unvgo/ghselfupdate" ) @@ -51,7 +51,7 @@ func handleUpdateCmd(ctx *ext.Context, u *ext.Update) error { return dispatcher.EndGroups } indocker := config.Docker == "true" - ctx.Sender.To(u.GetUserChat().AsInputPeer()).StyledText(ctx, html.String(nil, func() string { + ctx.Sender.To(u.EffectiveChat().GetInputPeer()).StyledText(ctx, html.String(nil, func() string { md := latest.ReleaseNotes md = regexp.MustCompile(`(?m)^###\s+   (.+)$`).ReplaceAllString(md, "$1") md = regexp.MustCompile(`(?m)^#####\s+    (.+)$`).ReplaceAllString(md, "$1") @@ -83,18 +83,12 @@ func handleUpdateCmd(ctx *ext.Context, u *ext.Update) error { "PublishedAt": latest.PublishedAt.Format("2006-01-02 15:04:05"), }) ctx.Reply(u, ext.ReplyTextString(text), &ext.ReplyOpts{ - Markup: &tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - &tg.KeyboardButtonCallback{ - Text: i18n.T(i18nk.BotMsgUpdateButtonUpgrade, nil), - Data: []byte("update"), - }, - }, - }, - }, - }, + Markup: markup.InlineKeyboard(markup.InlineButtonRow( + markup.Callback( + i18n.T(i18nk.BotMsgUpdateButtonUpgrade, nil), + []byte("update"), + ), + )), }) return dispatcher.EndGroups } @@ -107,7 +101,7 @@ func handleUpdateCallback(ctx *ext.Context, u *ext.Update) error { }))) return dispatcher.EndGroups } - ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(u), &tg.MessagesEditMessageRequest{ ID: u.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgUpdateInfoUpgradingWithVersion, map[string]any{ "Current": config.Version, @@ -115,7 +109,7 @@ func handleUpdateCallback(ctx *ext.Context, u *ext.Update) error { }) latest, err := selfupdate.Update(currentV) if err != nil { - ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(u), &tg.MessagesEditMessageRequest{ ID: u.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgUpdateErrorUpgradeFailed, map[string]any{ "Error": err.Error(), @@ -123,12 +117,12 @@ func handleUpdateCallback(ctx *ext.Context, u *ext.Update) error { }) return dispatcher.EndGroups } - ctx.EditMessage(u.GetUserChat().GetID(), &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID(u), &tg.MessagesEditMessageRequest{ ID: u.CallbackQuery.GetMsgID(), Message: i18n.T(i18nk.BotMsgUpdateInfoUpgradeSuccess, map[string]any{ "Version": latest.Version.String(), }), }) selfupdate.RequestRestart() - return errors.New("SAVEANTBOT-RESTART") + return selfupdate.ErrRestart } diff --git a/client/bot/handlers/utils/filters/url.go b/client/bot/handlers/utils/filters/url.go index 8de50ef1..6ab09e66 100644 --- a/client/bot/handlers/utils/filters/url.go +++ b/client/bot/handlers/utils/filters/url.go @@ -4,9 +4,9 @@ import ( "regexp" "slices" - "github.com/celestix/gotgproto/dispatcher/handlers/filters" - "github.com/celestix/gotgproto/types" "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/mygotg/dispatcher/handlers/filters" + "github.com/krau/mygotg/types" ) func RegexUrl(r *regexp.Regexp) filters.MessageFilter { diff --git a/client/bot/handlers/utils/mediautil/media.go b/client/bot/handlers/utils/mediautil/media.go index 88eaeb3a..8136bd33 100644 --- a/client/bot/handlers/utils/mediautil/media.go +++ b/client/bot/handlers/utils/mediautil/media.go @@ -49,36 +49,117 @@ func (f FilenameTemplateData) ToMap() map[string]string { func TfileOptions(ctx context.Context, user *database.User, message *tg.Message) []tfile.TGFileOption { opts := make([]tfile.TGFileOption, 0) - var fnameOpt tfile.TGFileOption + opts = append(opts, tfile.WithName(FilenameFor(ctx, user, message)), tfile.WithMessage(message)) + return opts +} + +// FilenameFor resolves the configured filename strategy for one message. +func FilenameFor(ctx context.Context, user *database.User, message *tg.Message) string { switch user.FilenameStrategy { case fnamest.Message.String(): - fnameOpt = tfile.WithName(tgutil.GenFileNameFromMessage(*message)) + return tgutil.GenFileNameFromMessage(*message) case fnamest.Template.String(): if user.FilenameTemplate == "" { log.FromContext(ctx).Warnf("empty filename template") - fnameOpt = tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*message)) - break + return tgutil.GenFileNameFromMessage(*message) } tmpl, err := template.New("filename").Parse(user.FilenameTemplate) if err != nil { log.FromContext(ctx).Errorf("failed to parse filename template: %s", err) - fnameOpt = tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*message)) - break + return tgutil.GenFileNameFromMessage(*message) } data := BuildFilenameTemplateData(message) var sb strings.Builder - err = tmpl.Execute(&sb, data) - if err != nil { + if err := tmpl.Execute(&sb, data); err != nil { log.FromContext(ctx).Errorf("failed to execute filename template: %s", err) - fnameOpt = tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*message)) - break + return tgutil.GenFileNameFromMessage(*message) + } + if sb.String() == "" { + return tgutil.GenFileNameFromMessage(*message) } - fnameOpt = tfile.WithName(sb.String()) + return sb.String() default: - fnameOpt = tfile.WithNameIfEmpty(tgutil.GenFileNameFromMessage(*message)) + if name := tgutil.GetOriginalFileName(message.Media); name != "" { + return name + } + return tgutil.GenFileNameFromMessage(*message) } - opts = append(opts, fnameOpt, tfile.WithMessage(message)) - return opts +} + +// ApplyAlbumCaptions shares one album caption with the items that carry none, +// so filename templates and message rules see the same text for every file in +// the group. +func ApplyAlbumCaptions(ctx context.Context, user *database.User, files []tfile.TGFileMessage) []tfile.TGFileMessage { + captions := make(map[int64]string) + for _, file := range files { + message := file.Message() + if message == nil { + continue + } + groupID, grouped := message.GetGroupedID() + if !grouped || groupID == 0 { + continue + } + if captions[groupID] == "" && strings.TrimSpace(message.GetMessage()) != "" { + captions[groupID] = message.GetMessage() + } + } + if len(captions) == 0 { + return files + } + // Track names already used in each group so a template without a unique + // field cannot collapse several album items onto the same path. + usedNames := make(map[int64]map[string]struct{}) + for _, file := range files { + message := file.Message() + if message == nil { + continue + } + groupID, grouped := message.GetGroupedID() + if !grouped || groupID == 0 { + continue + } + if usedNames[groupID] == nil { + usedNames[groupID] = make(map[string]struct{}) + } + usedNames[groupID][file.Name()] = struct{}{} + } + + result := make([]tfile.TGFileMessage, 0, len(files)) + for _, file := range files { + message := file.Message() + if message != nil { + if groupID, grouped := message.GetGroupedID(); grouped && groupID != 0 { + if caption := captions[groupID]; caption != "" && strings.TrimSpace(message.GetMessage()) == "" { + file = applyAlbumCaption(ctx, user, file, caption, usedNames[groupID]) + } + } + } + result = append(result, file) + } + return result +} + +type captionOverride struct { + tfile.TGFileMessage + message *tg.Message +} + +func (f *captionOverride) Message() *tg.Message { + return f.message +} + +func applyAlbumCaption(ctx context.Context, user *database.User, file tfile.TGFileMessage, caption string, used map[string]struct{}) tfile.TGFileMessage { + message := file.Message() + copied := *message + copied.Message = caption + if name := FilenameFor(ctx, user, &copied); name != "" { + if _, exists := used[name]; !exists { + used[name] = struct{}{} + file.SetName(name) + } + } + return &captionOverride{TGFileMessage: file, message: &copied} } func BuildFilenameTemplateData(message *tg.Message) map[string]string { @@ -99,7 +180,10 @@ func BuildFilenameTemplateData(message *tg.Message) map[string]string { }(), MsgGen: tgutil.GenFileNameFromMessage(*message), OrigName: func() string { - f, _ := tgutil.GetMediaFileName(message.Media) + f, err := tgutil.GetMediaFileName(message.Media) + if err != nil { + log.FromContext(context.Background()).Debugf("Failed to derive media filename: %v", err) + } return f }(), MsgDate: func() string { @@ -112,26 +196,17 @@ func BuildFilenameTemplateData(message *tg.Message) map[string]string { }(), MsgRaw: message.GetMessage(), ChatID: func() string { - // 如果消息是频道的(从消息链接中fetch的) 直接使用其chat id, - // 无论它是否是从其他来源转发的 - if message.GetPost() { - peer := message.GetPeerID() - switch p := peer.(type) { - case *tg.PeerChannel: - return intToStringOmitZero(p.ChannelID) - default: // impossible case - return intToStringOmitZero(tgutil.ChatIdFromPeer(peer)) + // For channel posts use the channel itself, regardless of the + // original forward source. Otherwise prefer the forward source. + peer := message.GetPeerID() + if !message.GetPost() { + if fwdHeader, ok := message.GetFwdFrom(); ok { + if fwdFrom, ok := fwdHeader.GetFromID(); ok { + peer = fwdFrom + } } } - fwdHeader, ok := message.GetFwdFrom() - if !ok { - return intToStringOmitZero(tgutil.ChatIdFromPeer(message.GetPeerID())) - } - fwdFrom, ok := fwdHeader.GetFromID() - if !ok { - return intToStringOmitZero(tgutil.ChatIdFromPeer(message.GetPeerID())) - } - return intToStringOmitZero(tgutil.ChatIdFromPeer(fwdFrom)) + return intToStringOmitZero(tgutil.ChatIdFromPeer(peer)) }(), }.ToMap() return data diff --git a/client/bot/handlers/utils/msgelem/dir.go b/client/bot/handlers/utils/msgelem/dir.go index c07daaba..def63a10 100644 --- a/client/bot/handlers/utils/msgelem/dir.go +++ b/client/bot/handlers/utils/msgelem/dir.go @@ -7,6 +7,7 @@ import ( "github.com/gotd/td/telegram/message/styling" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" ) @@ -27,7 +28,11 @@ func BuildDirHelpStyling(dirs []database.Dir) []styling.StyledTextOption { var sb strings.Builder for _, dir := range dirs { fmt.Fprintf(&sb, "%d: ", dir.ID) - sb.WriteString(dir.StorageName) + storageName := config.C().GetStorageNameByID(dir.StorageID) + if storageName == "" { + storageName = fmt.Sprintf("#%d", dir.StorageID) + } + sb.WriteString(storageName) sb.WriteString(" - ") sb.WriteString(dir.Path) sb.WriteString("\n") diff --git a/client/bot/handlers/utils/msgelem/rule.go b/client/bot/handlers/utils/msgelem/rule.go index 6c1f6ee4..18f20c5f 100644 --- a/client/bot/handlers/utils/msgelem/rule.go +++ b/client/bot/handlers/utils/msgelem/rule.go @@ -7,7 +7,9 @@ import ( "github.com/gotd/td/telegram/message/styling" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" + rulepkg "github.com/krau/SaveAny-Bot/pkg/rule" ) func BuildRuleHelpStyling(enabled bool, rules []database.Rule) []styling.StyledTextOption { @@ -32,7 +34,14 @@ func BuildRuleHelpStyling(enabled bool, rules []database.Rule) []styling.StyledT styling.Blockquote(func() string { var sb strings.Builder for _, rule := range rules { - ruleText := fmt.Sprintf("%s %s %s %s", rule.Type, rule.Data, rule.StorageName, rule.DirPath) + storageName := rulepkg.RuleStorNameChosen + if rule.StorageID != 0 { + storageName = config.C().GetStorageNameByID(rule.StorageID) + if storageName == "" { + storageName = fmt.Sprintf("#%d", rule.StorageID) + } + } + ruleText := fmt.Sprintf("%s %s %s %s", rule.Type, rule.Content, storageName, rule.DirPath) sb.WriteString(fmt.Sprintf("%d: %s\n", rule.ID, ruleText)) } return sb.String() diff --git a/client/bot/handlers/utils/msgelem/storage.go b/client/bot/handlers/utils/msgelem/storage.go index 9a0efb15..4fb90977 100644 --- a/client/bot/handlers/utils/msgelem/storage.go +++ b/client/bot/handlers/utils/msgelem/storage.go @@ -6,6 +6,7 @@ import ( "github.com/charmbracelet/log" "github.com/gotd/td/telegram/message/entity" + msgmarkup "github.com/gotd/td/telegram/message/markup" "github.com/gotd/td/telegram/message/styling" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/cache" @@ -33,13 +34,13 @@ func BuildAddSelectStorageKeyboard(stors []storage.Storage, adddata tcbdata.Add) } } - buttons := make([]tg.KeyboardButtonClass, 0) - for _, storage := range stors { + buttons := make([]tg.KeyboardInlineButton, 0) + for _, st := range stors { data := tcbdata.Add{ - TaskType: taskType, - SelectedStorName: storage.Name(), - SelectedDirPath: adddata.SelectedDirPath, - ConflictStrategy: adddata.ConflictStrategy, + TaskType: taskType, + SelectedStorageID: storage.ID(st), + SelectedDirPath: adddata.SelectedDirPath, + ConflictStrategy: adddata.ConflictStrategy, Files: adddata.Files, AsBatch: len(adddata.Files) > 1, @@ -56,23 +57,23 @@ func BuildAddSelectStorageKeyboard(stors []storage.Storage, adddata tcbdata.Add) YtdlpURLs: adddata.YtdlpURLs, YtdlpFlags: adddata.YtdlpFlags, - TransferSourceStorName: adddata.TransferSourceStorName, - TransferSourcePath: adddata.TransferSourcePath, - TransferFiles: adddata.TransferFiles, + TransferSourceStorageID: adddata.TransferSourceStorageID, + TransferSourcePath: adddata.TransferSourcePath, + TransferFiles: adddata.TransferFiles, } dataid := xid.New().String() err := cache.Set(dataid, data) if err != nil { return nil, err } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: storage.Name(), - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dataid), - }) + buttons = append(buttons, msgmarkup.Callback( + st.Name(), + fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dataid), + )) } markup := &tg.ReplyInlineMarkup{} for i := 0; i < len(buttons); i += 3 { - row := tg.KeyboardButtonRow{} + row := tg.KeyboardInlineButtonRow{} row.Buttons = buttons[i:min(i+3, len(buttons))] markup.Rows = append(markup.Rows, row) } @@ -121,7 +122,7 @@ func BuildConflictStrategyMarkup(adddata tcbdata.Add) (*tg.ReplyInlineMarkup, er {text: i18n.T(i18nk.BotMsgCommonButtonConflictOverwrite, nil), strategy: tcbdata.ConflictStrategyOverwrite}, {text: i18n.T(i18nk.BotMsgCommonButtonConflictSkip, nil), strategy: tcbdata.ConflictStrategySkip}, } - buttons := make([]tg.KeyboardButtonClass, 0, len(options)) + buttons := make([]tg.KeyboardInlineButton, 0, len(options)) for _, opt := range options { data := adddata data.ConflictStrategy = opt.strategy @@ -129,15 +130,15 @@ func BuildConflictStrategyMarkup(adddata tcbdata.Add) (*tg.ReplyInlineMarkup, er if err := cache.Set(dataid, data); err != nil { return nil, err } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: opt.text, - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dataid), - }) + buttons = append(buttons, msgmarkup.Callback( + opt.text, + fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dataid), + )) } - rows := make([]tg.KeyboardButtonRow, 0, len(buttons)) + rows := make([]tg.KeyboardInlineButtonRow, 0, len(buttons)) for _, button := range buttons { - rows = append(rows, tg.KeyboardButtonRow{ - Buttons: []tg.KeyboardButtonClass{button}, + rows = append(rows, tg.KeyboardInlineButtonRow{ + Buttons: []tg.KeyboardInlineButton{button}, }) } return &tg.ReplyInlineMarkup{Rows: rows}, nil @@ -146,24 +147,24 @@ func BuildConflictStrategyMarkup(adddata tcbdata.Add) (*tg.ReplyInlineMarkup, er func BuildSetDefaultStorageMarkup( ctx context.Context, stors []storage.Storage) (*tg.ReplyInlineMarkup, error) { - buttons := make([]tg.KeyboardButtonClass, 0) - for _, storage := range stors { + buttons := make([]tg.KeyboardInlineButton, 0) + for _, st := range stors { data := tcbdata.SetDefaultStorage{ - StorageName: storage.Name(), + StorageID: storage.ID(st), } dataid := xid.New().String() err := cache.Set(dataid, data) if err != nil { return nil, err } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: storage.Name(), - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeSetDefault, dataid), - }) + buttons = append(buttons, msgmarkup.Callback( + st.Name(), + fmt.Appendf(nil, "%s %s", tcbdata.TypeSetDefault, dataid), + )) } markup := &tg.ReplyInlineMarkup{} for i := 0; i < len(buttons); i += 3 { - row := tg.KeyboardButtonRow{} + row := tg.KeyboardInlineButtonRow{} row.Buttons = buttons[i:min(i+3, len(buttons))] markup.Rows = append(markup.Rows, row) } @@ -171,27 +172,27 @@ func BuildSetDefaultStorageMarkup( } func BuildSetDefaultDirMarkup(ctx context.Context, - seletedStorage string, + storageID uint, dirs []database.Dir) (*tg.ReplyInlineMarkup, error) { - buttons := make([]tg.KeyboardButtonClass, 0) + buttons := make([]tg.KeyboardInlineButton, 0) for _, dir := range dirs { dataid := xid.New().String() data := tcbdata.SetDefaultStorage{ - StorageName: seletedStorage, - DirID: dir.ID, + StorageID: storageID, + DirID: dir.ID, } err := cache.Set(dataid, data) if err != nil { return nil, err } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: dir.Path, - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeSetDefault, dataid), - }) + buttons = append(buttons, msgmarkup.Callback( + dir.Path, + fmt.Appendf(nil, "%s %s", tcbdata.TypeSetDefault, dataid), + )) } markup := &tg.ReplyInlineMarkup{} for i := 0; i < len(buttons); i += 3 { - row := tg.KeyboardButtonRow{} + row := tg.KeyboardInlineButtonRow{} row.Buttons = buttons[i:min(i+3, len(buttons))] markup.Rows = append(markup.Rows, row) } @@ -207,7 +208,7 @@ func BuildSetDirMarkupForAdd(dirs []database.Dir, dataid string) (*tg.ReplyInlin log.Warnf("Data already has a directory set: %d, %t", data.DirID, data.SettedDir) return nil, fmt.Errorf("data already has a directory set") } - buttons := make([]tg.KeyboardButtonClass, 0) + buttons := make([]tg.KeyboardInlineButton, 0) for _, dir := range dirs { dirDataId := xid.New().String() dirData := data @@ -217,10 +218,10 @@ func BuildSetDirMarkupForAdd(dirs []database.Dir, dataid string) (*tg.ReplyInlin if err != nil { return nil, fmt.Errorf("failed to set directory data in cache: %w", err) } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: dir.Path, - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dirDataId), - }) + buttons = append(buttons, msgmarkup.Callback( + dir.Path, + fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dirDataId), + )) } dirDefaultDataId := xid.New().String() dirDefaultData := data @@ -230,13 +231,13 @@ func BuildSetDirMarkupForAdd(dirs []database.Dir, dataid string) (*tg.ReplyInlin if err != nil { return nil, fmt.Errorf("failed to set default directory data in cache: %w", err) } - buttons = append(buttons, &tg.KeyboardButtonCallback{ - Text: i18n.T(i18nk.BotMsgDirButtonDefault, nil), - Data: fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dirDefaultDataId), - }) + buttons = append(buttons, msgmarkup.Callback( + i18n.T(i18nk.BotMsgDirButtonDefault, nil), + fmt.Appendf(nil, "%s %s", tcbdata.TypeAdd, dirDefaultDataId), + )) markup := &tg.ReplyInlineMarkup{} for i := 0; i < len(buttons); i += 3 { - row := tg.KeyboardButtonRow{} + row := tg.KeyboardInlineButtonRow{} row.Buttons = buttons[i:min(i+3, len(buttons))] markup.Rows = append(markup.Rows, row) } diff --git a/client/bot/handlers/utils/ruleutil/rule.go b/client/bot/handlers/utils/ruleutil/rule.go index 87fa252e..a3691afb 100644 --- a/client/bot/handlers/utils/ruleutil/rule.go +++ b/client/bot/handlers/utils/ruleutil/rule.go @@ -6,6 +6,7 @@ import ( "github.com/duke-git/lancet/v2/convertor" "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/rule" "github.com/krau/SaveAny-Bot/pkg/tfile" @@ -54,9 +55,17 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m } logger := log.FromContext(ctx) for _, ur := range rules { + storageName := rule.RuleStorNameChosen + if ur.StorageID != 0 { + storageName = config.C().GetStorageNameByID(ur.StorageID) + if storageName == "" { + logger.Warnf("Skipping rule %d: unknown storage ID %d", ur.ID, ur.StorageID) + continue + } + } switch ur.Type { case rule.FileNameRegex.String(): - ru, err := rule.NewRuleFileNameRegex(ur.StorageName, ur.DirPath, ur.Data) + ru, err := rule.NewRuleFileNameRegex(storageName, ur.DirPath, ur.Content) if err != nil { logger.Errorf("Failed to create rule: %s", err) continue @@ -71,7 +80,7 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m matchedStorageName = matchedStorName(ru.StorageName()) } case rule.MessageRegex.String(): - ru, err := rule.NewRuleMessageRegex(ur.StorageName, ur.DirPath, ur.Data) + ru, err := rule.NewRuleMessageRegex(storageName, ur.DirPath, ur.Content) if err != nil { logger.Errorf("Failed to create rule: %s", err) continue @@ -86,11 +95,11 @@ func ApplyRule(ctx context.Context, rules []database.Rule, inputs *ruleInput) (m matchedStorageName = matchedStorName(ru.StorageName()) } case rule.IsAlbum.String(): - matchAlbum, err := convertor.ToBool(ur.Data) + matchAlbum, err := convertor.ToBool(ur.Content) if err != nil { matchAlbum = false } - ru, err := rule.NewRuleMediaType(ur.StorageName, ur.DirPath, matchAlbum) + ru, err := rule.NewRuleMediaType(storageName, ur.DirPath, matchAlbum) if err != nil { logger.Errorf("Failed to create rule: %s", err) continue diff --git a/client/bot/handlers/utils/shortcut/aria2.go b/client/bot/handlers/utils/shortcut/aria2.go index 66fda149..1ebab060 100644 --- a/client/bot/handlers/utils/shortcut/aria2.go +++ b/client/bot/handlers/utils/shortcut/aria2.go @@ -1,8 +1,6 @@ package shortcut import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/i18n" @@ -12,10 +10,12 @@ import ( "github.com/krau/SaveAny-Bot/core/tasks/aria2dl" "github.com/krau/SaveAny-Bot/pkg/aria2" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) -func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, uris []string, aria2Client *aria2.Client, msgID int, userID int64) error { +func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, uris []string, aria2Client *aria2.Client, msgID int, chatID int64) error { logger := log.FromContext(ctx) injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) @@ -24,9 +24,9 @@ func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa if len(uris) == 0 { logger.Error("URIs list is empty") - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, - Message: i18n.T(i18nk.BotMsgDlErrorNoValidLinks, nil), + Message: i18n.T(i18nk.BotMsgDownloadErrorNoValidLinks, nil), }) return dispatcher.EndGroups } @@ -34,7 +34,7 @@ func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa gid, err := aria2Client.AddURI(ctx, uris, nil) if err != nil { logger.Errorf("Failed to add aria2 download: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgAria2ErrorAddingAria2Download, map[string]any{ "Error": err.Error(), @@ -44,10 +44,10 @@ func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa } logger.Infof("Aria2 download added with GID: %s", gid) - task := aria2dl.NewTask(xid.New().String(), injectCtx, gid, uris, aria2Client, stor, dirPath, aria2dl.NewProgress(msgID, userID)) + task := aria2dl.NewTask(xid.New().String(), injectCtx, gid, uris, aria2Client, stor, dirPath, aria2dl.NewProgress(msgID, chatID)) if err := core.AddTask(injectCtx, task); err != nil { logger.Errorf("Failed to add task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -55,7 +55,7 @@ func CreateAndAddAria2TaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa }) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonInfoTaskAdded, nil), }) diff --git a/client/bot/handlers/utils/shortcut/directlinks.go b/client/bot/handlers/utils/shortcut/directlinks.go index 9fc0bf3b..34f316bc 100644 --- a/client/bot/handlers/utils/shortcut/directlinks.go +++ b/client/bot/handlers/utils/shortcut/directlinks.go @@ -1,8 +1,6 @@ package shortcut import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/i18n" @@ -11,15 +9,17 @@ import ( "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/core/tasks/directlinks" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) -func CreateAndAddDirectTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, links []string, msgID int, userID int64) error { +func CreateAndAddDirectTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, links []string, msgID int, chatID int64) error { injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) - task := directlinks.NewTask(xid.New().String(), injectCtx, links, stor, dirPath, directlinks.NewProgress(msgID, userID)) + task := directlinks.NewTask(xid.New().String(), injectCtx, links, stor, dirPath, directlinks.NewProgress(msgID, chatID)) if err := core.AddTask(injectCtx, task); err != nil { log.FromContext(ctx).Errorf("Failed to add task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -27,7 +27,7 @@ func CreateAndAddDirectTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirP }) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ Message: i18n.T(i18nk.BotMsgCommonInfoTaskAdded, nil), }) return dispatcher.EndGroups diff --git a/client/bot/handlers/utils/shortcut/message.go b/client/bot/handlers/utils/shortcut/message.go index ef3c5dd6..2b12bc67 100644 --- a/client/bot/handlers/utils/shortcut/message.go +++ b/client/bot/handlers/utils/shortcut/message.go @@ -2,14 +2,8 @@ package shortcut import ( - "encoding/json" - "fmt" "net/url" - "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" - "github.com/celestix/gotgproto/types" "github.com/charmbracelet/log" "github.com/gotd/td/telegram/downloader" "github.com/gotd/td/tg" @@ -20,12 +14,17 @@ import ( "github.com/krau/SaveAny-Bot/common/cache" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/mediafilter" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/common/utils/tphutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/mediatype" "github.com/krau/SaveAny-Bot/pkg/telegraph" "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" + "github.com/krau/mygotg/types" ) func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message *tg.Message, tfileopts ...tfile.TGFileOption) (replied *types.Message, @@ -33,10 +32,12 @@ func GetFileFromMessageWithReply(ctx *ext.Context, update *ext.Update, message * ) { logger := log.FromContext(ctx) media := message.Media - supported := mediautil.IsSupported(media) - if !supported { + if !mediautil.IsSupported(media) { return nil, nil, dispatcher.ContinueGroups } + if !mediafilter.Allowed(media) { + return nil, nil, dispatcher.EndGroups + } replied, err = ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonInfoFetchingFileInfo, nil)), nil) if err != nil { @@ -77,7 +78,7 @@ func GetFilesFromUpdateLinkMessageWithReplyEdit(ctx *ext.Context, update *ext.Up logger.Errorf("failed to edit message: %s", err) } } - user, err := database.GetUserByChatID(ctx, update.GetUserChat().GetID()) + user, err := database.GetUserByChatID(ctx, tgutil.SenderID(update)) if err != nil { logger.Errorf("failed to get user from db: %s", err) editReplied(i18n.T(i18nk.BotMsgCommonErrorGetUserInfoFailed, map[string]any{ @@ -96,6 +97,10 @@ func GetFilesFromUpdateLinkMessageWithReplyEdit(ctx *ext.Context, update *ext.Up logger.Debugf("message %d has no media", msg.GetID()) return } + if !mediautil.IsSupported(media) || !mediafilter.Allowed(media) { + logger.Debugf("message %d media type is filtered", msg.GetID()) + return + } opts := mediautil.TfileOptions(ctx, user, msg) file, err := tfile.FromMediaMessage(media, client, msg, opts...) if err != nil { @@ -169,12 +174,16 @@ type TelegraphResult struct { func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*types.Message, *TelegraphResult, error) { logger := log.FromContext(ctx) + if !mediafilter.Allows(mediatype.Image) { + ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorNoSavableFilesFound)), nil) + return nil, nil, dispatcher.EndGroups + } tphurl := findTelegraphURL(update.EffectiveMessage.Message) if tphurl == "" { logger.Warnf("No telegraph url found but called handleTelegraph") return nil, nil, dispatcher.ContinueGroups } - pagepath, err := parseTelegraphPagePath(tphurl) + pagepath, err := tphutil.ParsePagePath(tphurl) if err != nil { logger.Errorf("Failed to parse telegraph path: %s", err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorParseTelegraphPathFailed, map[string]any{ @@ -198,32 +207,7 @@ func GetTphPicsFromMessageWithReply(ctx *ext.Context, update *ext.Update) (*type } imgs := make([]string, 0) for _, elem := range page.Content { - var node telegraph.NodeElement - data, err := json.Marshal(elem) - if err != nil { - logger.Errorf("Failed to marshal element: %s", err) - continue - } - err = json.Unmarshal(data, &node) - if err != nil { - logger.Errorf("Failed to unmarshal element: %s", err) - continue - } - - if len(node.Children) != 0 { - for _, child := range node.Children { - imgs = append(imgs, tphutil.GetNodeImages(child)...) - } - } - if node.Tag == "img" { - if src, ok := node.Attrs["src"]; ok { - if strings.HasPrefix(src, "/file/") { - // handle images on telegra.ph server - src = "https://telegra.ph" + src - } - imgs = append(imgs, src) - } - } + imgs = append(imgs, tphutil.GetNodeImages(elem)...) } if len(imgs) == 0 { logger.Warn("No images found in telegraph page") @@ -247,7 +231,7 @@ func findTelegraphURL(msg *tg.Message) string { if firstMatch == "" { firstMatch = tphurl } - if _, err := parseTelegraphPagePath(tphurl); err == nil { + if _, err := tphutil.ParsePagePath(tphurl); err == nil { return tphurl } } @@ -263,26 +247,3 @@ func findTelegraphURL(msg *tg.Message) string { } return firstMatch } - -func parseTelegraphPagePath(pageURL string) (string, error) { - u, err := url.Parse(pageURL) - if err != nil { - return "", fmt.Errorf("invalid telegraph URL: %w", err) - } - if u.Scheme != "https" || !strings.EqualFold(u.Hostname(), "telegra.ph") { - return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host) - } - pagepath := strings.Trim(u.EscapedPath(), "/") - if pagepath == "" || strings.Contains(pagepath, "/") { - return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path) - } - pagepath, err = url.PathUnescape(pagepath) - if err != nil { - return "", fmt.Errorf("failed to unescape telegraph path: %w", err) - } - pagepath = strings.TrimSpace(pagepath) - if pagepath == "" || strings.Contains(pagepath, "/") { - return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path) - } - return pagepath, nil -} diff --git a/client/bot/handlers/utils/shortcut/message_telegraph_test.go b/client/bot/handlers/utils/shortcut/message_telegraph_test.go index 1567575c..5071a877 100644 --- a/client/bot/handlers/utils/shortcut/message_telegraph_test.go +++ b/client/bot/handlers/utils/shortcut/message_telegraph_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/common/utils/tphutil" ) func TestFindTelegraphURL(t *testing.T) { @@ -145,18 +146,18 @@ func TestParseTelegraphPagePath(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseTelegraphPagePath(tt.pageURL) + got, err := tphutil.ParsePagePath(tt.pageURL) if tt.wantErr { if err == nil { - t.Fatalf("parseTelegraphPagePath(%q) returned no error", tt.pageURL) + t.Fatalf("tphutil.ParsePagePath(%q) returned no error", tt.pageURL) } return } if err != nil { - t.Fatalf("parseTelegraphPagePath(%q) failed: %v", tt.pageURL, err) + t.Fatalf("tphutil.ParsePagePath(%q) failed: %v", tt.pageURL, err) } if got != tt.want { - t.Fatalf("parseTelegraphPagePath(%q) = %q, want %q", tt.pageURL, got, tt.want) + t.Fatalf("tphutil.ParsePagePath(%q) = %q, want %q", tt.pageURL, got, tt.want) } }) } diff --git a/client/bot/handlers/utils/shortcut/parsed.go b/client/bot/handlers/utils/shortcut/parsed.go index 12863d9a..d27f71ce 100644 --- a/client/bot/handlers/utils/shortcut/parsed.go +++ b/client/bot/handlers/utils/shortcut/parsed.go @@ -1,8 +1,6 @@ package shortcut import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" @@ -13,15 +11,17 @@ import ( parsed "github.com/krau/SaveAny-Bot/core/tasks/parsed" "github.com/krau/SaveAny-Bot/pkg/parser" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) -func CreateAndAddParsedTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, item *parser.Item, msgID int, userID int64) error { +func CreateAndAddParsedTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, item *parser.Item, msgID int, chatID int64) error { injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) - task := parsed.NewTask(xid.New().String(), injectCtx, stor, dirPath, item, parsed.NewProgress(msgID, userID)) + task := parsed.NewTask(xid.New().String(), injectCtx, stor, dirPath, item, parsed.NewProgress(msgID, chatID)) if err := core.AddTask(injectCtx, task); err != nil { log.FromContext(ctx).Errorf("Failed to add task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -30,7 +30,7 @@ func CreateAndAddParsedTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirP return dispatcher.EndGroups } text, entities := msgelem.BuildTaskAddedEntities(ctx, item.Title, core.GetLength(ctx)) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: text, Entities: entities, diff --git a/client/bot/handlers/utils/shortcut/tftask.go b/client/bot/handlers/utils/shortcut/tftask.go index d6f5d5c0..5ec7e947 100644 --- a/client/bot/handlers/utils/shortcut/tftask.go +++ b/client/bot/handlers/utils/shortcut/tftask.go @@ -1,19 +1,20 @@ package shortcut import ( + "fmt" "path" "strings" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/conflictutil" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/ruleutil" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/core/tasks/batchtfile" tftask "github.com/krau/SaveAny-Bot/core/tasks/tfile" @@ -22,16 +23,18 @@ import ( "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) -func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage.Storage, dirPath string, file tfile.TGFileMessage, trackMsgID int, conflictStrategy ...string) error { +func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID, chatID int64, stor storage.Storage, dirPath string, file tfile.TGFileMessage, trackMsgID int, conflictStrategy ...string) error { logger := log.FromContext(ctx) strategy := selectedConflictStrategy(conflictStrategy) user, err := database.GetUserByChatID(ctx, userID) if err != nil { logger.Errorf("Failed to get user by chat ID: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorGetUserWithErrFailed, map[string]any{ "Error": err.Error(), @@ -52,7 +55,7 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage stor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, matchedStorageName.String()) if err != nil { logger.Errorf("Failed to get storage by user ID and name: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ "Error": err.Error(), @@ -65,12 +68,15 @@ func CreateAndAddTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage startCreateTask: storagePath := path.Join(dirPath, file.Name()) if strategy == tcbdata.ConflictStrategyAsk || strategy == tcbdata.ConflictStrategySkip { - exists := stor.Exists(ctx, storagePath) + exists, err := stor.Exists(ctx, storagePath) + if err != nil { + return fmt.Errorf("failed to check storage path %s: %w", storagePath, err) + } if exists && strategy == tcbdata.ConflictStrategyAsk { - return promptTGFileConflictStrategy(ctx, userID, stor.Name(), dirPath, []tfile.TGFileMessage{file}, false, []string{conflictutil.FormatPath(stor.Name(), storagePath)}, trackMsgID) + return promptTGFileConflictStrategy(ctx, chatID, stor.Name(), dirPath, []tfile.TGFileMessage{file}, false, []string{conflictutil.FormatPath(stor.Name(), storagePath)}, trackMsgID) } if exists { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonInfoAllConflictFilesSkipped, map[string]any{ "Skipped": file.Name(), @@ -88,10 +94,10 @@ startCreateTask: task, err := tftask.NewTGFileTask(taskid, injectCtx, file, stor, storagePath, tftask.NewProgressTrack( trackMsgID, - userID)) + chatID)) if err != nil { logger.Errorf("create task failed: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskCreateFailed, map[string]any{ "Error": err.Error(), @@ -101,7 +107,7 @@ startCreateTask: } if err := core.AddTask(injectCtx, task); err != nil { logger.Errorf("add task failed: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -110,7 +116,7 @@ startCreateTask: return dispatcher.EndGroups } text, entities := msgelem.BuildTaskAddedEntities(ctx, file.Name(), core.GetLength(injectCtx)) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: text, Entities: entities, @@ -119,13 +125,13 @@ startCreateTask: return dispatcher.EndGroups } -func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor storage.Storage, dirPath string, files []tfile.TGFileMessage, trackMsgID int, conflictStrategy ...string) error { +func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID, chatID int64, stor storage.Storage, dirPath string, files []tfile.TGFileMessage, trackMsgID int, conflictStrategy ...string) error { logger := log.FromContext(ctx) strategy := selectedConflictStrategy(conflictStrategy) user, err := database.GetUserByChatID(ctx, userID) if err != nil { logger.Errorf("Failed to get user by chat ID: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorGetUserWithErrFailed, map[string]any{ "Error": err.Error(), @@ -134,6 +140,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st return dispatcher.EndGroups } strategy = conflictutil.ResolveStrategy(user, strategy) + files = mediautil.ApplyAlbumCaptions(ctx, user, files) useRule := user.ApplyRule && user.Rules != nil @@ -168,7 +175,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st fileStor, err = storage.GetStorageByUserIDAndName(ctx, user.ChatID, storName) if err != nil { logger.Errorf("Failed to get storage by user ID and name: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorGetStorageFailed, map[string]any{ "Error": err.Error(), @@ -180,7 +187,10 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st if !matchedDirPath.NeedNewForAlbum() { storPath := path.Join(matchedDirPath.String(), file.Name()) if strategy == tcbdata.ConflictStrategyAsk || strategy == tcbdata.ConflictStrategySkip { - exists := fileStor.Exists(ctx, storPath) + exists, err := fileStor.Exists(ctx, storPath) + if err != nil { + return fmt.Errorf("failed to check storage path %s: %w", storPath, err) + } if exists && strategy == tcbdata.ConflictStrategyAsk { conflicts = append(conflicts, conflictutil.FormatPath(fileStor.Name(), storPath)) continue @@ -193,7 +203,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st elem, err := batchtfile.NewTaskElement(fileStor, storPath, file) if err != nil { logger.Errorf("Failed to create task element: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskCreateFailed, map[string]any{ "Error": err.Error(), @@ -223,7 +233,35 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st } } for _, afiles := range albumFiles { - if len(afiles) <= 1 { + if len(afiles) == 1 { + af := afiles[0] + storPath := path.Join(af.dirPath, af.file.Name()) + if strategy == tcbdata.ConflictStrategyAsk || strategy == tcbdata.ConflictStrategySkip { + exists, err := af.storage.Exists(ctx, storPath) + if err != nil { + return fmt.Errorf("failed to check storage path %s: %w", storPath, err) + } + if exists && strategy == tcbdata.ConflictStrategyAsk { + conflicts = append(conflicts, conflictutil.FormatPath(af.storage.Name(), storPath)) + continue + } + if exists { + skipped = append(skipped, af.file.Name()) + continue + } + } + elem, err := batchtfile.NewTaskElement(af.storage, storPath, af.file) + if err != nil { + logger.Errorf("Failed to create task element for album file: %s", err) + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ + ID: trackMsgID, + Message: i18n.T(i18nk.BotMsgCommonErrorTaskCreateFailed, map[string]any{ + "Error": err.Error(), + }), + }) + return dispatcher.EndGroups + } + elems = append(elems, *elem) continue } // 对于需要新建目录的文件, 将第一个文件的文件名(去除扩展名)作为目录名 @@ -233,7 +271,10 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st for _, af := range afiles { afstorPath := path.Join(af.dirPath, albumDir, af.file.Name()) if strategy == tcbdata.ConflictStrategyAsk || strategy == tcbdata.ConflictStrategySkip { - exists := albumStor.Exists(ctx, afstorPath) + exists, err := albumStor.Exists(ctx, afstorPath) + if err != nil { + return fmt.Errorf("failed to check storage path %s: %w", afstorPath, err) + } if exists && strategy == tcbdata.ConflictStrategyAsk { conflicts = append(conflicts, conflictutil.FormatPath(albumStor.Name(), afstorPath)) continue @@ -246,7 +287,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st elem, err := batchtfile.NewTaskElement(albumStor, afstorPath, af.file) if err != nil { logger.Errorf("Failed to create task element for album file: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskCreateFailed, map[string]any{ "Error": err.Error(), @@ -259,7 +300,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st } if strategy == tcbdata.ConflictStrategyAsk && len(conflicts) > 0 { - return promptTGFileConflictStrategy(ctx, userID, stor.Name(), dirPath, files, true, conflicts, trackMsgID) + return promptTGFileConflictStrategy(ctx, chatID, stor.Name(), dirPath, files, true, conflicts, trackMsgID) } injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) @@ -267,7 +308,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st injectCtx = storage.WithOverwrite(injectCtx) } if len(elems) == 0 { - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonInfoAllConflictFilesSkipped, map[string]any{ "Skipped": strings.Join(skipped, "\n"), @@ -277,10 +318,10 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st return dispatcher.EndGroups } taskid := xid.New().String() - task := batchtfile.NewBatchTGFileTask(taskid, injectCtx, elems, batchtfile.NewProgressTrackerWithSkipped(trackMsgID, userID, skipped), true) + task := batchtfile.NewBatchTGFileTask(taskid, injectCtx, elems, batchtfile.NewProgressTrackerWithSkipped(trackMsgID, chatID, skipped), true) if err := core.AddTask(injectCtx, task); err != nil { logger.Errorf("Failed to add batch task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -288,7 +329,7 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st }) return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: buildBatchAddedMessage(len(elems), skipped), ReplyMarkup: nil, @@ -296,19 +337,19 @@ func CreateAndAddBatchTGFileTaskWithEdit(ctx *ext.Context, userID int64, stor st return dispatcher.EndGroups } -func promptTGFileConflictStrategy(ctx *ext.Context, userID int64, storageName, dirPath string, files []tfile.TGFileMessage, asBatch bool, conflicts []string, trackMsgID int) error { +func promptTGFileConflictStrategy(ctx *ext.Context, chatID int64, storageName, dirPath string, files []tfile.TGFileMessage, asBatch bool, conflicts []string, trackMsgID int) error { markup, err := msgelem.BuildConflictStrategyMarkup(tcbdata.Add{ - TaskType: tasktype.TaskTypeTgfiles, - SelectedStorName: storageName, - SettedDir: true, - SelectedDirPath: dirPath, - Files: files, - AsBatch: asBatch, + TaskType: tasktype.TaskTypeTgfiles, + SelectedStorageID: config.C().GetStorageIDByName(storageName), + SettedDir: true, + SelectedDirPath: dirPath, + Files: files, + AsBatch: asBatch, }) if err != nil { return err } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonPromptSelectConflictStrategy, map[string]any{"Files": conflictutil.FormatPaths(conflicts)}), ReplyMarkup: markup, diff --git a/client/bot/handlers/utils/shortcut/tphtask.go b/client/bot/handlers/utils/shortcut/tphtask.go index c2d0d54b..5fd5fd7f 100644 --- a/client/bot/handlers/utils/shortcut/tphtask.go +++ b/client/bot/handlers/utils/shortcut/tphtask.go @@ -1,8 +1,6 @@ package shortcut import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" @@ -14,12 +12,14 @@ import ( tphtask "github.com/krau/SaveAny-Bot/core/tasks/telegraph" "github.com/krau/SaveAny-Bot/pkg/telegraph" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) func CreateAndAddtelegraphWithEdit( ctx *ext.Context, - userID int64, + chatID int64, tphpage *telegraph.Page, dirPath string, // unescaped ph path for file storage pics []string, @@ -34,11 +34,11 @@ func CreateAndAddtelegraphWithEdit( stor, dirPath, tphutil.DefaultClient(), - tphtask.NewProgress(trackMsgID, userID), + tphtask.NewProgress(trackMsgID, chatID), ) if err := core.AddTask(injectCtx, task); err != nil { log.FromContext(ctx).Errorf("Failed to add task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -47,7 +47,7 @@ func CreateAndAddtelegraphWithEdit( return dispatcher.EndGroups } text, entities := msgelem.BuildTaskAddedEntities(ctx, tphpage.Title, core.GetLength(ctx)) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: trackMsgID, Message: text, Entities: entities, diff --git a/client/bot/handlers/utils/shortcut/ytdlp.go b/client/bot/handlers/utils/shortcut/ytdlp.go index 2c2325e1..245c16be 100644 --- a/client/bot/handlers/utils/shortcut/ytdlp.go +++ b/client/bot/handlers/utils/shortcut/ytdlp.go @@ -1,10 +1,10 @@ package shortcut import ( - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" "github.com/krau/SaveAny-Bot/common/i18n" @@ -15,15 +15,15 @@ import ( "github.com/krau/SaveAny-Bot/storage" ) -func CreateAndAddYtdlpTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, urls []string, flags []string, msgID int, userID int64) error { +func CreateAndAddYtdlpTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPath string, urls []string, flags []string, msgID int, chatID int64) error { logger := log.FromContext(ctx) injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) if len(urls) == 0 { logger.Error("URLs list is empty") - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, - Message: i18n.T(i18nk.BotMsgYtdlpErrorNoValidUrls, nil), + Message: i18n.T(i18nk.BotMsgDownloadErrorNoValidLinks, nil), }) return dispatcher.EndGroups } @@ -37,12 +37,12 @@ func CreateAndAddYtdlpTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa flags, stor, dirPath, - ytdlp.NewProgress(msgID, userID), + ytdlp.NewProgress(msgID, chatID), ) if err := core.AddTask(injectCtx, task); err != nil { logger.Errorf("Failed to add yt-dlp task: %s", err) - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonErrorTaskAddFailed, map[string]any{ "Error": err.Error(), @@ -51,7 +51,7 @@ func CreateAndAddYtdlpTaskWithEdit(ctx *ext.Context, stor storage.Storage, dirPa return dispatcher.EndGroups } - ctx.EditMessage(userID, &tg.MessagesEditMessageRequest{ + ctx.EditMessage(chatID, &tg.MessagesEditMessageRequest{ ID: msgID, Message: i18n.T(i18nk.BotMsgCommonInfoTaskAdded, nil), }) diff --git a/client/bot/handlers/watch.go b/client/bot/handlers/watch.go index da62720a..16af1e83 100644 --- a/client/bot/handlers/watch.go +++ b/client/bot/handlers/watch.go @@ -1,30 +1,37 @@ package handlers import ( + "context" "fmt" "path" "regexp" + "strconv" "strings" "sync" "text/template" "time" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/message/markup" + "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/mediautil" + "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/ruleutil" userclient "github.com/krau/SaveAny-Bot/client/user" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" + "github.com/krau/SaveAny-Bot/common/mediafilter" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" coretfile "github.com/krau/SaveAny-Bot/core/tasks/tfile" "github.com/krau/SaveAny-Bot/database" "github.com/krau/SaveAny-Bot/pkg/enums/fnamest" + "github.com/krau/SaveAny-Bot/pkg/tcbdata" "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) @@ -32,20 +39,17 @@ func handleWatchCmd(ctx *ext.Context, update *ext.Update) error { logger := log.FromContext(ctx) args := strings.Split(update.EffectiveMessage.Text, " ") if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchHelpText)), nil) + ctx.Reply(update, ext.ReplyTextString(watchMenuText("")), &ext.ReplyOpts{Markup: watchMenuMarkup()}) return dispatcher.EndGroups } - userChatID := update.GetUserChat().GetID() + userChatID := responsibleUserID(update) user, err := database.GetUserByChatID(ctx, userChatID) if err != nil { logger.Errorf("Failed to get user: %s", err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetUserFailed)), nil) return dispatcher.EndGroups } - if args[1] == "notify" { - return handleWatchNotifyCmd(ctx, update, user, args[2:]) - } - if user.DefaultStorage == "" { + if user.DefaultStorageID == 0 { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorDefaultStorageNotSet)), nil) return dispatcher.EndGroups } @@ -67,9 +71,8 @@ func handleWatchCmd(ctx *ext.Context, update *ext.Update) error { filter := "" if len(args) > 2 { filterArg := strings.Join(args[2:], " ") - filterType, _, _ := strings.Cut(filterArg, ":") - filterData := strings.Split(filterArg, ":")[1] - if filterType == "" || filterData == "" { + filterType, filterData, ok := strings.Cut(filterArg, ":") + if !ok || filterType == "" || filterData == "" { ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchErrorFilterFormatInvalid)), nil) return dispatcher.EndGroups } @@ -87,9 +90,9 @@ func handleWatchCmd(ctx *ext.Context, update *ext.Update) error { } } if err := user.WatchChat(ctx, database.WatchChat{ - UserID: user.ID, - ChatID: chatID, - Filter: filter, + UserID: user.ID, + ChatID: chatID, + MessageFilter: filter, }); err != nil { logger.Errorf("Failed to watch chat %d: %s", chatID, err) ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchErrorWatchChatFailed, map[string]any{"Error": err.Error()})), nil) @@ -99,63 +102,171 @@ func handleWatchCmd(ctx *ext.Context, update *ext.Update) error { return dispatcher.EndGroups } -func handleLswatchCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - userChatID := update.GetUserChat().GetID() - user, err := database.GetUserByChatID(ctx, userChatID) - if err != nil { - logger.Errorf("Failed to get user: %s", err) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetUserFailed)), nil) - return dispatcher.EndGroups +func watchMenuText(extra string) string { + text := i18n.T(i18nk.BotMsgWatchMenuTitle) + "\n\n" + i18n.T(i18nk.BotMsgWatchMenuHint) + if extra != "" { + text += "\n\n" + extra + } + return text +} + +func watchMenuMarkup() *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{ + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuAdd), fmt.Appendf(nil, "%s add", tcbdata.TypeWatch)), + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuList), fmt.Appendf(nil, "%s list", tcbdata.TypeWatch)), + }}, + {Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuNotify), fmt.Appendf(nil, "%s notify", tcbdata.TypeWatch)), + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuSync), fmt.Appendf(nil, "%s sync", tcbdata.TypeWatch)), + }}, + }} +} + +func watchBackMarkup() *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{Rows: []tg.KeyboardInlineButtonRow{{ + Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuBack), fmt.Appendf(nil, "%s menu", tcbdata.TypeWatch)), + }, + }}} +} + +func handleWatchCallback(ctx *ext.Context, update *ext.Update) error { + args := strings.Fields(string(update.CallbackQuery.Data)) + if len(args) < 2 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) } - chats := user.WatchChats + switch args[1] { + case "menu": + return editWatchMenu(ctx, update) + case "add": + return editWatchAdd(ctx, update) + case "list": + return editWatchList(ctx, update) + case "notify": + return editWatchNotify(ctx, update) + case "sync": + return editWatchSync(ctx, update) + case "rm": + return editWatchRemove(ctx, update, args) + default: + return editWatchMenu(ctx, update) + } +} + +func editWatchMenu(ctx *ext.Context, update *ext.Update) error { + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: watchMenuText(""), + ReplyMarkup: watchMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func editWatchAdd(ctx *ext.Context, update *ext.Update) error { + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: watchMenuText(i18n.T(i18nk.BotMsgWatchMenuAddHelp)), + ReplyMarkup: watchBackMarkup(), + }) + return dispatcher.EndGroups +} + +func watchListText(chats []database.WatchChat) string { if len(chats) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchInfoWatchListEmpty)), nil) - return dispatcher.EndGroups + return i18n.T(i18nk.BotMsgWatchInfoWatchListEmpty) } - var sb strings.Builder - sb.WriteString(i18n.T(i18nk.BotMsgWatchInfoWatchListHeader)) + var b strings.Builder + b.WriteString(i18n.T(i18nk.BotMsgWatchInfoWatchListHeader)) for _, chat := range chats { - sb.WriteString("- ") - sb.WriteString(fmt.Sprintf("%d", chat.ChatID)) - if chat.Filter != "" { - sb.WriteString(i18n.T(i18nk.BotMsgWatchInfoWatchListFilterPrefix)) - sb.WriteString(chat.Filter) - sb.WriteString(")") + fmt.Fprintf(&b, "\n- %d", chat.ChatID) + if chat.MessageFilter != "" { + b.WriteString(i18n.T(i18nk.BotMsgWatchInfoWatchListFilterPrefix)) + b.WriteString(chat.MessageFilter) + b.WriteString(")") } - sb.WriteString("\n") } - ctx.Reply(update, ext.ReplyTextString(sb.String()), nil) + return b.String() +} + +func watchListMarkup(chats []database.WatchChat) *tg.ReplyInlineMarkup { + rows := make([]tg.KeyboardInlineButtonRow, 0, len(chats)+1) + for _, chat := range chats { + rows = append(rows, tg.KeyboardInlineButtonRow{Buttons: []tg.KeyboardInlineButton{ + markup.Callback(i18n.T(i18nk.BotMsgWatchMenuStop), fmt.Appendf(nil, "%s rm %d", tcbdata.TypeWatch, chat.ChatID)), + }}) + } + rows = append(rows, watchBackMarkup().Rows...) + return &tg.ReplyInlineMarkup{Rows: rows} +} + +func editWatchList(ctx *ext.Context, update *ext.Update) error { + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: watchListText(user.WatchChats), + ReplyMarkup: watchListMarkup(user.WatchChats), + }) return dispatcher.EndGroups } -func handleUnwatchCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - args := strings.Split(update.EffectiveMessage.Text, " ") - if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchErrorUnwatchNoChatProvided)), nil) - return dispatcher.EndGroups +func editWatchNotify(ctx *ext.Context, update *ext.Update) error { + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err } - userChatID := update.GetUserChat().GetID() - user, err := database.GetUserByChatID(ctx, userChatID) + user.WatchNotify = !user.WatchNotify + if err := database.UpdateUser(ctx, user); err != nil { + return err + } + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: watchMenuText(i18n.T(watchNotifyStateKey(user.WatchNotify))), + ReplyMarkup: watchMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func editWatchSync(ctx *ext.Context, update *ext.Update) error { + count, err := syncPeers(ctx) if err != nil { - logger.Errorf("Failed to get user: %s", err) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorGetUserFailed)), nil) + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: i18n.T(i18nk.BotMsgSyncpeersFailed, map[string]any{"Error": err.Error()}), + }) return dispatcher.EndGroups } - chatArg := args[1] - chatID, err := tgutil.ParseChatID(ctx, chatArg) + ctx.EditMessage(chatID(update), &tg.MessagesEditMessageRequest{ + ID: update.CallbackQuery.GetMsgID(), + Message: watchMenuText(i18n.T(i18nk.BotMsgSyncpeersSuccess, map[string]any{"Count": count})), + ReplyMarkup: watchMenuMarkup(), + }) + return dispatcher.EndGroups +} + +func editWatchRemove(ctx *ext.Context, update *ext.Update, args []string) error { + if len(args) < 3 { + return fmt.Errorf("invalid callback data: %q", update.CallbackQuery.Data) + } + chatID, err := strconv.ParseInt(args[2], 10, 64) if err != nil { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorInvalidIdOrUsername, map[string]any{"Error": err.Error()})), nil) - return dispatcher.EndGroups + return err + } + userID := update.CallbackQuery.GetUserID() + user, err := database.GetUserByChatID(ctx, userID) + if err != nil { + return err } if err := user.UnwatchChat(ctx, chatID); err != nil { - logger.Errorf("Failed to unwatch chat %d: %s", chatID, err) - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchErrorUnwatchChatFailed, map[string]any{"Error": err.Error()})), nil) + ctx.AnswerCallback(msgelem.AlertCallbackAnswer(update.CallbackQuery.GetQueryID(), i18n.T(i18nk.BotMsgWatchErrorUnwatchChatFailed, map[string]any{"Error": err.Error()}))) return dispatcher.EndGroups } - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchInfoWatchChatStopped, map[string]any{"Chat": chatArg})), nil) - return dispatcher.EndGroups + return editWatchList(ctx, update) } type watchMediaGroupHandler struct { @@ -205,13 +316,24 @@ func (w *watchMediaGroupHandler) addFile(chatID int64, userID uint, file tfile.T }) } -func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent, botCtx *ext.Context) { - if userclient.GetCtx() == nil { - return - } - logger := log.FromContext(userclient.GetCtx()) - for event := range ch { +// ListenUserbotMediaEvents consumes userbot media events until the channel is +// closed. resolve returns the client context used for notifications. +func ListenUserbotMediaEvents(runCtx context.Context, resolve func() *ext.Context) { + logger := log.FromContext(runCtx) + for event := range userclient.GetMediaMessageCh() { + if event.File == nil || event.Ctx == nil { + continue + } + botCtx := resolve() + if botCtx == nil { + continue + } logger.Debug("Received media message event", "chat_id", event.ChatID, "file_name", event.File.Name()) + message := event.File.Message() + if message == nil || !mediafilter.Allowed(message.Media) { + logger.Debug("Skipping filtered media message", "chat_id", event.ChatID, "file_name", event.File.Name()) + continue + } ctx := event.Ctx file := event.File chats, err := database.GetWatchChatsByChatID(ctx, event.ChatID) @@ -219,10 +341,10 @@ func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent, botCtx *ext.C logger.Errorf("Failed to get watch chats for chat ID %d: %v", event.ChatID, err) continue } - msgText := event.File.Message().GetMessage() + msgText := message.GetMessage() for _, chat := range chats { - if chat.Filter != "" { - filter := strings.Split(chat.Filter, ":") + if chat.MessageFilter != "" { + filter := strings.Split(chat.MessageFilter, ":") if len(filter) != 2 { logger.Warnf("Invalid filter format in chat %d, skipping", chat.ChatID) continue @@ -246,18 +368,18 @@ func listenMediaMessageEvent(ch chan userclient.MediaMessageEvent, botCtx *ext.C logger.Errorf("Failed to get user by ID %d: %v", chat.UserID, err) continue } - if user.DefaultStorage == "" { + if user.DefaultStorageID == 0 { logger.Warnf("User %d has no default storage set, skipping media message handling", chat.UserID) continue } - stor, err := storage.GetStorageByUserIDAndName(ctx, user.ChatID, user.DefaultStorage) + stor, err := storage.GetStorageByUserIDAndID(ctx, user.ChatID, user.DefaultStorageID) if err != nil { - logger.Errorf("Failed to get storage by user ID %d and name %s: %v", user.ChatID, user.DefaultStorage, err) + logger.Errorf("Failed to get storage ID %d for user %d: %v", user.DefaultStorageID, user.ChatID, err) continue } var defaultDirPath string - if user.DefaultDir != 0 { - dir, err := database.GetDirByID(ctx, user.DefaultDir) + if user.DefaultDirID != 0 { + dir, err := database.GetDirByID(ctx, user.DefaultDirID) if err != nil { logger.Warnf("Failed to get default dir for user %d: %v, using root", user.ChatID, err) } else { @@ -343,6 +465,7 @@ func processWatchMediaGroup(ctx *ext.Context, botCtx *ext.Context, user *databas } useRule := user.ApplyRule && user.Rules != nil + files = mediautil.ApplyAlbumCaptions(ctx, user, files) applyRule := func(file tfile.TGFileMessage) (string, ruleutil.MatchedDirPath) { if !useRule { @@ -404,7 +527,19 @@ func processWatchMediaGroup(ctx *ext.Context, botCtx *ext.Context, user *databas injectCtx := tgutil.ExtWithContext(ctx.Context, ctx) totalTasks := 0 for groupID, afiles := range albumFiles { - if len(afiles) <= 1 { + if len(afiles) == 1 { + af := afiles[0] + taskid := xid.New().String() + task, err := coretfile.NewTGFileTask(taskid, injectCtx, af.file, af.storage, path.Join(af.dirPath, af.file.Name()), newWatchNotifyProgress(ctx, botCtx, user)) + if err != nil { + logger.Errorf("create task failed for album file: %s", err) + continue + } + if err := core.AddTask(injectCtx, task); err != nil { + logger.Errorf("add task failed: %s", err) + continue + } + totalTasks++ continue } diff --git a/client/bot/handlers/watch_notify.go b/client/bot/handlers/watch_notify.go index c1a0c3ec..a1c8c370 100644 --- a/client/bot/handlers/watch_notify.go +++ b/client/bot/handlers/watch_notify.go @@ -2,13 +2,11 @@ package handlers import ( "context" - "strings" "sync" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" "github.com/gotd/td/tg" + "github.com/krau/mygotg/ext" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" @@ -17,39 +15,6 @@ import ( "github.com/krau/SaveAny-Bot/database" ) -// handleWatchNotifyCmd toggles the task notifications for watched chats and -// reports the current state, which is disabled by default. -func handleWatchNotifyCmd(ctx *ext.Context, update *ext.Update, user *database.User, args []string) error { - if len(args) > 0 { - enabled, ok := parseWatchNotifyArg(args[0]) - if !ok { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgWatchErrorNotifyArgInvalid)), nil) - return dispatcher.EndGroups - } - user.WatchNotify = enabled - if err := database.UpdateUser(ctx, user); err != nil { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgCommonErrorUpdateUserInfoFailed, map[string]any{ - "Error": err.Error(), - })), nil) - return dispatcher.EndGroups - } - } - ctx.Reply(update, ext.ReplyTextString(i18n.T(watchNotifyStateKey(user.WatchNotify))), nil) - return dispatcher.EndGroups -} - -// parseWatchNotifyArg parses the argument of /watch notify. -func parseWatchNotifyArg(arg string) (enabled bool, ok bool) { - switch strings.ToLower(arg) { - case "on": - return true, true - case "off": - return false, true - default: - return false, false - } -} - func watchNotifyStateKey(enabled bool) i18nk.Key { if enabled { return i18nk.BotMsgWatchInfoNotifyEnabled diff --git a/client/bot/handlers/watch_notify_test.go b/client/bot/handlers/watch_notify_test.go index eddf167e..a4ba5371 100644 --- a/client/bot/handlers/watch_notify_test.go +++ b/client/bot/handlers/watch_notify_test.go @@ -7,8 +7,8 @@ import ( "strings" "testing" - "github.com/celestix/gotgproto/ext" "github.com/gotd/td/tg" + "github.com/krau/mygotg/ext" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/utils/tgutil" @@ -16,30 +16,6 @@ import ( "github.com/krau/SaveAny-Bot/database" ) -func TestParseWatchNotifyArg(t *testing.T) { - tests := []struct { - arg string - wantEnabled bool - wantOK bool - }{ - {"on", true, true}, - {"ON", true, true}, - {"off", false, true}, - {"Off", false, true}, - {"", false, false}, - {"yes", false, false}, - {"-1002229835658", false, false}, - } - for _, tt := range tests { - t.Run(tt.arg, func(t *testing.T) { - enabled, ok := parseWatchNotifyArg(tt.arg) - if enabled != tt.wantEnabled || ok != tt.wantOK { - t.Errorf("parseWatchNotifyArg(%q) = (%v, %v), want (%v, %v)", tt.arg, enabled, ok, tt.wantEnabled, tt.wantOK) - } - }) - } -} - type notifyCtxKey struct{} type notifyTestInfo struct{} diff --git a/client/bot/handlers/ytdlp.go b/client/bot/handlers/ytdlp.go index 1f9c8edd..2d235367 100644 --- a/client/bot/handlers/ytdlp.go +++ b/client/bot/handlers/ytdlp.go @@ -5,53 +5,9 @@ import ( "strings" "unicode" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/charmbracelet/log" - - "github.com/krau/SaveAny-Bot/client/bot/handlers/utils/msgelem" - "github.com/krau/SaveAny-Bot/common/i18n" - "github.com/krau/SaveAny-Bot/common/i18n/i18nk" - "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" - "github.com/krau/SaveAny-Bot/pkg/tcbdata" - "github.com/krau/SaveAny-Bot/storage" ) -func handleYtdlpCmd(ctx *ext.Context, update *ext.Update) error { - logger := log.FromContext(ctx) - args := splitQuotedArgs(update.EffectiveMessage.Text) - if len(args) < 2 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgYtdlpUsage)), nil) - return dispatcher.EndGroups - } - - urls, flags := parseYtdlpArgs(logger, args[1:]) - - if len(urls) == 0 { - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgYtdlpErrorNoValidUrls)), nil) - return dispatcher.EndGroups - } - - logger.Debugf("Preparing yt-dlp download for %d URL(s) with %d flag(s)", len(urls), len(flags)) - - markup, err := msgelem.BuildAddSelectStorageKeyboard(storage.GetUserStorages(ctx, update.GetUserChat().GetID()), tcbdata.Add{ - TaskType: tasktype.TaskTypeYtdlp, - YtdlpURLs: urls, - YtdlpFlags: flags, - }) - if err != nil { - return err - } - - ctx.Reply(update, ext.ReplyTextString(i18n.T(i18nk.BotMsgYtdlpInfoUrlsSelectStorage, map[string]any{ - "Count": len(urls), - })), &ext.ReplyOpts{ - Markup: markup, - }) - - return dispatcher.EndGroups -} - // splitQuotedArgs splits a command line on whitespace, keeping quoted values // together and removing the quotes. Quotes only delimit a value when they start // one, so apostrophes inside arguments stay literal. @@ -84,7 +40,7 @@ func splitQuotedArgs(text string) []string { return args } -// parseYtdlpArgs separates the /ytdlp arguments into URLs and custom yt-dlp +// parseYtdlpArgs separates download arguments into URLs and custom yt-dlp // flags. A flag value is kept together with its flag, unless it looks like a URL // of its own. func parseYtdlpArgs(logger *log.Logger, args []string) (urls []string, flags []string) { @@ -110,10 +66,15 @@ func parseYtdlpArgs(logger *log.Logger, args []string) (urls []string, flags []s } } else { u, err := url.Parse(arg) - if err != nil || u.Scheme == "" || u.Host == "" { + if err != nil || u.Scheme == "" { logger.Warnf("Invalid URL: %s", arg) continue } + // Magnet and other non-web schemes are valid Aria2 inputs but not + // yt-dlp URLs; parseDownloadURIs handles them separately. + if u.Host == "" { + continue + } urls = append(urls, arg) } } diff --git a/client/bot/handlers/ytdlp_test.go b/client/bot/handlers/ytdlp_test.go index d9a6ec9a..fccdc221 100644 --- a/client/bot/handlers/ytdlp_test.go +++ b/client/bot/handlers/ytdlp_test.go @@ -16,79 +16,79 @@ func TestYtdlpArgumentParsing(t *testing.T) { }{ { name: "Single URL without flags", - input: "/ytdlp https://example.com/video", + input: "/download https://example.com/video", expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{}, }, { name: "Multiple URLs without flags", - input: "/ytdlp https://example.com/v1 https://example.com/v2", + input: "/download https://example.com/v1 https://example.com/v2", expectedURLs: []string{"https://example.com/v1", "https://example.com/v2"}, expectedFlags: []string{}, }, { name: "URL with format flag", - input: "/ytdlp --format best https://example.com/video", + input: "/download --format best https://example.com/video", expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"--format", "best"}, }, { name: "URL with extract-audio flag", - input: "/ytdlp --extract-audio --audio-format mp3 https://example.com/video", + input: "/download --extract-audio --audio-format mp3 https://example.com/video", expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"--extract-audio", "--audio-format", "mp3"}, }, { name: "Multiple URLs with flags", - input: "/ytdlp --format best https://example.com/v1 https://example.com/v2", + input: "/download --format best https://example.com/v1 https://example.com/v2", expectedURLs: []string{"https://example.com/v1", "https://example.com/v2"}, expectedFlags: []string{"--format", "best"}, }, { name: "Flags mixed with URLs", - input: "/ytdlp https://example.com/v1 --format best https://example.com/v2", + input: "/download https://example.com/v1 --format best https://example.com/v2", expectedURLs: []string{"https://example.com/v1", "https://example.com/v2"}, expectedFlags: []string{"--format", "best"}, }, { name: "Short flag", - input: "/ytdlp -f best https://example.com/video", + input: "/download -f best https://example.com/video", expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"-f", "best"}, }, { name: "Boolean flag", - input: "/ytdlp --extract-audio https://example.com/video", + input: "/download --extract-audio https://example.com/video", expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"--extract-audio"}, }, { name: "Output template flag with spaces", - input: `/ytdlp -o "%(uploader)s - %(title)s.%(ext)s" https://example.com/video`, + input: `/download -o "%(uploader)s - %(title)s.%(ext)s" https://example.com/video`, expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"-o", "%(uploader)s - %(title)s.%(ext)s"}, }, { name: "Single quoted template", - input: `/ytdlp -o '%(title)s.%(ext)s' https://example.com/video`, + input: `/download -o '%(title)s.%(ext)s' https://example.com/video`, expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"-o", "%(title)s.%(ext)s"}, }, { name: "Apostrophe inside a URL", - input: `/ytdlp https://en.wikipedia.org/wiki/O'Brien -f best`, + input: `/download https://en.wikipedia.org/wiki/O'Brien -f best`, expectedURLs: []string{"https://en.wikipedia.org/wiki/O'Brien"}, expectedFlags: []string{"-f", "best"}, }, { name: "Apostrophe inside a quoted template", - input: `/ytdlp -o "%(title)s 'live'.%(ext)s" https://example.com/video`, + input: `/download -o "%(title)s 'live'.%(ext)s" https://example.com/video`, expectedURLs: []string{"https://example.com/video"}, expectedFlags: []string{"-o", "%(title)s 'live'.%(ext)s"}, }, { name: "Invalid URL is dropped", - input: "/ytdlp not-a-url -f best", + input: "/download not-a-url -f best", expectedURLs: []string{}, expectedFlags: []string{"-f", "best"}, }, diff --git a/client/middleware/floodwait.go b/client/middleware/floodwait.go deleted file mode 100644 index 4c553c41..00000000 --- a/client/middleware/floodwait.go +++ /dev/null @@ -1,19 +0,0 @@ -package middleware - -import ( - "time" - - "github.com/gotd/contrib/middleware/floodwait" - "github.com/gotd/contrib/middleware/ratelimit" - "github.com/gotd/td/telegram" - "golang.org/x/time/rate" -) - -func NewFloodWaitMiddlewares(maxRetries uint) []telegram.Middleware { - waiter := floodwait.NewSimpleWaiter().WithMaxRetries(maxRetries) - ratelimiter := ratelimit.New(rate.Every(time.Millisecond*100), 5) - return []telegram.Middleware{ - waiter, - ratelimiter, - } -} diff --git a/client/middleware/retry/retry.go b/client/middleware/retry/retry.go index b13b0863..026f519d 100644 --- a/client/middleware/retry/retry.go +++ b/client/middleware/retry/retry.go @@ -28,11 +28,13 @@ type retry struct { func (r retry) Handle(next tg.Invoker) telegram.InvokeFunc { return func(ctx context.Context, input bin.Encoder, output bin.Decoder) error { retries := 0 + var lastErr error for retries < r.max { if err := next.Invoke(ctx, input, output); err != nil { if tgerr.Is(err, r.errors...) { log.FromContext(ctx).Debug("retry middleware", "retries", retries, "error", err) + lastErr = err retries++ continue } @@ -42,7 +44,10 @@ func (r retry) Handle(next tg.Invoker) telegram.InvokeFunc { return nil } - return fmt.Errorf("retry limit reached after %d attempts", r.max) + if lastErr == nil { + return fmt.Errorf("retry limit reached after %d attempts", r.max) + } + return fmt.Errorf("retry limit reached after %d attempts: %w", r.max, lastErr) } } diff --git a/client/relay/download.go b/client/relay/download.go new file mode 100644 index 00000000..dfa4635f --- /dev/null +++ b/client/relay/download.go @@ -0,0 +1,417 @@ +package relay + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/gotd/td/constant" + "github.com/gotd/td/telegram/downloader" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/worker" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/ext" +) + +var runningClients = worker.RunningClients + +// Prepare wraps a Telegram file with a relay-backed downloader when a relay +// channel and at least two running helpers are available. Any failure leaves +// the original single-client file in place. +func Prepare(ctx context.Context, file tfile.TGFile) (tfile.TGFile, func(context.Context)) { + return prepare(ctx, file, "") +} + +// PrepareResumable is Prepare with a sidecar state file. The state lets a +// crashed download reuse its relay message instead of forwarding it again. +func PrepareResumable(ctx context.Context, file tfile.TGFile, statePath string) (tfile.TGFile, func(context.Context)) { + return prepare(ctx, file, statePath) +} + +func prepare(ctx context.Context, file tfile.TGFile, statePath string) (tfile.TGFile, func(context.Context)) { + relayChatID := config.C().Telegram.RelayChatID + if relayChatID == 0 { + return file, nil + } + if config.C().Stream { + log.FromContext(ctx).Debug("Relay download skipped: stream mode is enabled") + return file, nil + } + minSize := int64(config.C().RelayMinSizeMB) << 20 + if file == nil || file.Size() < minSize { + return file, nil + } + messageFile, ok := file.(tfile.TGFileMessage) + if !ok || messageFile.Message() == nil { + log.FromContext(ctx).Debug("Relay download skipped: file is not a Telegram message") + return file, nil + } + extCtx := tgutil.ExtFromContext(ctx) + if extCtx == nil { + return file, nil + } + workers := runningClients() + if len(workers) < 2 { + log.FromContext(ctx).Debugf("Relay download skipped: %d running helper bot(s), need at least 2", len(workers)) + return file, nil + } + relayPeer, err := extCtx.ResolveInputPeerById(relayChatID) + if err != nil { + log.FromContext(ctx).Debugf("Relay channel is unavailable, using primary client: %v", err) + return file, nil + } + if _, ok := relayPeer.(*tg.InputPeerChannel); !ok { + log.FromContext(ctx).Warn("Relay chat is not a channel, using primary client") + return file, nil + } + message := messageFile.Message() + sourceChatID := tgutil.ChatIdFromPeer(message.GetPeerID()) + if sourceChatID == 0 || message.GetID() == 0 { + return file, nil + } + identity := tfile.Identity(file.Location()) + keepMessages := config.C().Telegram.RelayKeepMessages + + if statePath != "" { + state, stateErr := loadResumeState(statePath) + if stateErr != nil { + log.FromContext(ctx).Debugf("Relay resume state unavailable: %v", stateErr) + } else if state != nil { + if state.RelayChatID == relayChatID && state.matches(identity) { + relayed := newRelayFile(messageFile, extCtx, relayChatID, state.RelayMessageID, workers, keepMessages, statePath) + accessible := relayed.accessibleWorkers(ctx) + if len(accessible) >= 2 { + relayed.workers = accessible + log.FromContext(ctx).Infof( + "Relay download reused: file=%q helpers=%s message_id=%d", + file.Name(), workerNames(accessible), state.RelayMessageID, + ) + return relayed, relayed.cleanup + } + log.FromContext(ctx).Warnf("Relay resume message %d is no longer readable; forwarding again", state.RelayMessageID) + relayed.cleanup(ctx) + } else { + log.FromContext(ctx).Debug("Relay resume state belongs to another file; discarding") + stale := newRelayFile(messageFile, extCtx, state.RelayChatID, state.RelayMessageID, workers, keepMessages, statePath) + stale.cleanup(ctx) + } + } + } + + updates, err := extCtx.ForwardMessages(sourceChatID, relayChatID, &tg.MessagesForwardMessagesRequest{ + ID: []int{message.GetID()}, + }) + if err != nil { + log.FromContext(ctx).Debugf("Relay forward failed, using primary client: %v", err) + return file, nil + } + relayMessageID := forwardedMessageID(updates, relayChatID) + if relayMessageID == 0 { + log.FromContext(ctx).Warn("Relay forward returned no channel message ID, using primary client") + return file, nil + } + relayed := newRelayFile(messageFile, extCtx, relayChatID, relayMessageID, workers, keepMessages, statePath) + accessible := relayed.accessibleWorkers(ctx) + if len(accessible) < 2 { + log.FromContext(ctx).Warnf("Relay download skipped: only %d helper bot(s) can read the relay channel", len(accessible)) + relayed.cleanup(ctx) + return file, nil + } + relayed.workers = accessible + if statePath != "" { + state := resumeState{ + Version: resumeStateVersion, + RelayChatID: relayChatID, + RelayMessageID: relayMessageID, + SourceKind: identity.Kind, + SourceID: identity.ID, + SourceAccessHash: identity.AccessHash, + } + if err := saveResumeState(statePath, state); err != nil { + log.FromContext(ctx).Warnf("Failed to persist relay resume state: %v", err) + relayed.cleanup(ctx) + return file, nil + } + } + log.FromContext(ctx).Infof( + "Relay download enabled: file=%q size=%d helpers=%s message_id=%d keep_messages=%t", + file.Name(), file.Size(), workerNames(accessible), relayMessageID, relayed.keepMessages, + ) + return relayed, relayed.cleanup +} + +func newRelayFile( + base tfile.TGFileMessage, + primary *ext.Context, + relayChatID int64, + relayMessageID int, + workers []worker.ClientRef, + keepMessages bool, + statePath string, +) *relayFile { + return &relayFile{ + base: base, + primary: primary, + relayChatID: relayChatID, + relayMessageID: relayMessageID, + workers: workers, + keepMessages: keepMessages, + statePath: statePath, + locations: make(map[uint]tg.InputFileLocationClass, len(workers)), + served: make(map[uint]int, len(workers)), + } +} + +func (f *relayFile) accessibleWorkers(ctx context.Context) []worker.ClientRef { + accessible := make([]worker.ClientRef, 0, len(f.workers)) + for _, ref := range f.workers { + if _, err := f.workerLocation(ctx, ref); err != nil { + log.FromContext(ctx).Debugf("Relay worker %s cannot read channel: %v", ref.Name, err) + continue + } + accessible = append(accessible, ref) + } + return accessible +} + +func workerNames(workers []worker.ClientRef) string { + names := make([]string, 0, len(workers)) + for _, ref := range workers { + if ref.Name != "" { + names = append(names, ref.Name) + } + } + return strings.Join(names, ", ") +} + +type relayFile struct { + base tfile.TGFileMessage + primary *ext.Context + relayChatID int64 + relayMessageID int + workers []worker.ClientRef + keepMessages bool + statePath string + + mu sync.Mutex + next int + locations map[uint]tg.InputFileLocationClass + served map[uint]int +} + +func (f *relayFile) Location() tg.InputFileLocationClass { return f.base.Location() } +func (f *relayFile) Dler() downloader.Client { return &multiClient{file: f, base: f.base.Dler()} } +func (f *relayFile) Size() int64 { return f.base.Size() } +func (f *relayFile) Name() string { return f.base.Name() } +func (f *relayFile) SetName(name string) { f.base.SetName(name) } +func (f *relayFile) Message() *tg.Message { return f.base.Message() } +func (f *relayFile) WorkerCount() int { return len(f.workers) } + +func (f *relayFile) cleanup(ctx context.Context) { + summary := f.servedSummary() + if summary != "" { + log.FromContext(ctx).Infof("Relay download finished: %s", summary) + } + if err := removeResumeState(f.statePath); err != nil { + log.FromContext(ctx).Warnf("Failed to remove relay resume state: %v", err) + } + if f.keepMessages { + log.FromContext(ctx).Debugf("Relay message %d kept by configuration", f.relayMessageID) + return + } + peer, err := f.primary.ResolveInputPeerById(f.relayChatID) + if err != nil { + return + } + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + switch p := peer.(type) { + case *tg.InputPeerChannel: + _, err = f.primary.Raw.ChannelsDeleteMessages(cleanupCtx, &tg.ChannelsDeleteMessagesRequest{ + Channel: &tg.InputChannel{ChannelID: p.ChannelID, AccessHash: p.AccessHash}, + ID: []int{f.relayMessageID}, + }) + case *tg.InputPeerChat, *tg.InputPeerUser: + _, err = f.primary.Raw.MessagesDeleteMessages(cleanupCtx, &tg.MessagesDeleteMessagesRequest{ + Revoke: true, + ID: []int{f.relayMessageID}, + }) + } + if err != nil { + log.FromContext(ctx).Warnf("Relay message %d cleanup failed: %v", f.relayMessageID, err) + } +} + +func (f *relayFile) recordServed(workerID uint) { + f.mu.Lock() + defer f.mu.Unlock() + if f.served == nil { + f.served = make(map[uint]int) + } + f.served[workerID]++ +} + +func (f *relayFile) servedSummary() string { + f.mu.Lock() + defer f.mu.Unlock() + parts := make([]string, 0, len(f.workers)) + for _, ref := range f.workers { + if count := f.served[ref.ID]; count > 0 { + parts = append(parts, fmt.Sprintf("%s=%d chunk(s)", ref.Name, count)) + } + } + return strings.Join(parts, ", ") +} + +type multiClient struct { + file *relayFile + base downloader.Client +} + +func (c *multiClient) UploadGetFile(ctx context.Context, req *tg.UploadGetFileRequest) (tg.UploadFileClass, error) { + for range c.file.workers { + ref := c.file.nextWorker() + location, err := c.file.workerLocation(ctx, ref) + if err != nil { + log.FromContext(ctx).Debugf("Relay worker %s location failed: %v", ref.Name, err) + continue + } + workerReq := *req + workerReq.Location = location + result, err := ref.Ext.Raw.UploadGetFile(ctx, &workerReq) + if err == nil { + c.file.recordServed(ref.ID) + log.FromContext(ctx).Debugf("Relay chunk served: worker=%s offset=%d limit=%d", ref.Name, req.Offset, req.Limit) + return result, nil + } + log.FromContext(ctx).Debugf("Relay worker %s download failed: %v", ref.Name, err) + c.file.invalidate(ref.ID) + } + log.FromContext(ctx).Debugf("Relay chunk fallback to primary: offset=%d limit=%d", req.Offset, req.Limit) + return c.base.UploadGetFile(ctx, req) +} + +func (c *multiClient) UploadGetFileHashes(ctx context.Context, req *tg.UploadGetFileHashesRequest) ([]tg.FileHash, error) { + return c.base.UploadGetFileHashes(ctx, req) +} + +func (c *multiClient) UploadReuploadCDNFile(ctx context.Context, req *tg.UploadReuploadCDNFileRequest) ([]tg.FileHash, error) { + return c.base.UploadReuploadCDNFile(ctx, req) +} + +func (c *multiClient) UploadGetCDNFileHashes(ctx context.Context, req *tg.UploadGetCDNFileHashesRequest) ([]tg.FileHash, error) { + return c.base.UploadGetCDNFileHashes(ctx, req) +} + +func (c *multiClient) UploadGetWebFile(ctx context.Context, req *tg.UploadGetWebFileRequest) (*tg.UploadWebFile, error) { + return c.base.UploadGetWebFile(ctx, req) +} + +func (f *relayFile) nextWorker() worker.ClientRef { + f.mu.Lock() + defer f.mu.Unlock() + ref := f.workers[f.next%len(f.workers)] + f.next++ + return ref +} + +func (f *relayFile) workerLocation(ctx context.Context, ref worker.ClientRef) (tg.InputFileLocationClass, error) { + f.mu.Lock() + location, ok := f.locations[ref.ID] + f.mu.Unlock() + if ok { + return location, nil + } + peer, err := ref.Ext.ResolveInputPeerById(f.relayChatID) + if err != nil { + return nil, err + } + inputChannel, ok := peer.(*tg.InputPeerChannel) + if !ok { + return nil, fmt.Errorf("relay chat is not a channel") + } + result, err := ref.Ext.Raw.ChannelsGetMessages(ctx, &tg.ChannelsGetMessagesRequest{ + Channel: &tg.InputChannel{ChannelID: inputChannel.ChannelID, AccessHash: inputChannel.AccessHash}, + ID: []tg.InputMessageClass{&tg.InputMessageID{ID: f.relayMessageID}}, + }) + if err != nil { + return nil, err + } + message := firstMessage(result) + if message == nil || message.Media == nil { + return nil, errors.New("relay message has no media") + } + relayed, err := tfile.FromMedia(message.Media, ref.Ext.Raw, tfile.WithMessage(message), tfile.WithName(f.base.Name())) + if err != nil { + return nil, err + } + location = relayed.Location() + f.mu.Lock() + f.locations[ref.ID] = location + f.mu.Unlock() + return location, nil +} + +func (f *relayFile) invalidate(workerID uint) { + f.mu.Lock() + delete(f.locations, workerID) + f.mu.Unlock() +} + +func firstMessage(messages tg.MessagesMessagesClass) *tg.Message { + var list []tg.MessageClass + switch m := messages.(type) { + case *tg.MessagesMessages: + list = m.Messages + case *tg.MessagesMessagesSlice: + list = m.Messages + case *tg.MessagesChannelMessages: + list = m.Messages + default: + return nil + } + for _, item := range list { + if message, ok := item.(*tg.Message); ok { + return message + } + } + return nil +} + +func forwardedMessageID(updates tg.UpdatesClass, relayChatID int64) int { + var list []tg.UpdateClass + switch u := updates.(type) { + case *tg.Updates: + list = u.Updates + case *tg.UpdatesCombined: + list = u.Updates + default: + return 0 + } + for _, update := range list { + var messageClass tg.MessageClass + switch u := update.(type) { + case *tg.UpdateNewChannelMessage: + messageClass = u.Message + case *tg.UpdateNewMessage: + messageClass = u.Message + default: + continue + } + message, ok := messageClass.(*tg.Message) + if !ok { + continue + } + if tgutil.ChatIdFromPeer(message.GetPeerID()) == relayChatID || + tgutil.ChatIdFromPeer(message.GetPeerID()) == constant.TDLibPeerID(relayChatID).ToPlain() { + return message.GetID() + } + } + return 0 +} diff --git a/client/relay/download_test.go b/client/relay/download_test.go new file mode 100644 index 00000000..cc40cefc --- /dev/null +++ b/client/relay/download_test.go @@ -0,0 +1,126 @@ +package relay + +import ( + "context" + "sync" + "testing" + + "github.com/gotd/td/bin" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/worker" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/ext" + tgstorage "github.com/krau/mygotg/storage" +) + +type relayFakeInvoker struct { + mu sync.Mutex + forwards int + deletes int + relayMsg *tg.Message + relayDoc *tg.Document +} + +func (f *relayFakeInvoker) Invoke(_ context.Context, input bin.Encoder, output bin.Decoder) error { + switch req := input.(type) { + case *tg.MessagesForwardMessagesRequest: + f.mu.Lock() + f.forwards++ + f.mu.Unlock() + if box, ok := output.(*tg.UpdatesBox); ok { + box.Updates = &tg.Updates{Updates: []tg.UpdateClass{&tg.UpdateNewChannelMessage{Message: f.relayMsg}}} + } + case *tg.ChannelsGetMessagesRequest: + if box, ok := output.(*tg.MessagesMessagesBox); ok { + box.Messages = &tg.MessagesChannelMessages{Messages: []tg.MessageClass{f.relayMsg}} + } + case *tg.ChannelsDeleteMessagesRequest: + f.mu.Lock() + f.deletes++ + f.mu.Unlock() + case *tg.UploadGetFileRequest: + if box, ok := output.(*tg.UploadFileBox); ok { + box.File = &tg.UploadFile{Bytes: make([]byte, req.Limit)} + } + } + return nil +} + +func relayTestExt(inv *relayFakeInvoker, peers []int64) *ext.Context { + storage := tgstorage.NewPeerStorage(nil, true) + for _, id := range peers { + storage.AddPeer(id, 1, tgstorage.TypeChannel, "") + } + return ext.NewContext(context.Background(), tg.NewClient(inv), storage, &tg.User{ID: 1}, nil, nil, false) +} + +func TestPrepareResumableReusesRelayMessage(t *testing.T) { + oldCfg := config.C() + cfg := oldCfg + cfg.Telegram.RelayChatID = -1000000000500 + cfg.Telegram.RelayKeepMessages = false + config.Set(cfg) + defer config.Set(oldCfg) + + const sourceChatID int64 = -1000000000400 + doc := &tg.Document{ID: 7, AccessHash: 8, FileReference: []byte{1}, Size: 9 << 20, DCID: 2} + relayMsg := &tg.Message{ID: 777, PeerID: &tg.PeerChannel{ChannelID: 500}, Media: &tg.MessageMediaDocument{Document: doc}} + sourceMsg := &tg.Message{ID: 10, PeerID: &tg.PeerChannel{ChannelID: sourceChatID}, Media: &tg.MessageMediaDocument{Document: doc}} + primaryInv := &relayFakeInvoker{relayMsg: relayMsg, relayDoc: doc} + primary := relayTestExt(primaryInv, []int64{sourceChatID, 500}) + worker1 := &relayFakeInvoker{relayMsg: relayMsg, relayDoc: doc} + worker2 := &relayFakeInvoker{relayMsg: relayMsg, relayDoc: doc} + oldRunning := runningClients + runningClients = func() []worker.ClientRef { + return []worker.ClientRef{ + {ID: 1, Name: "one", Ext: relayTestExt(worker1, []int64{500})}, + {ID: 2, Name: "two", Ext: relayTestExt(worker2, []int64{500})}, + } + } + defer func() { runningClients = oldRunning }() + + file, err := tfile.FromMediaMessage(sourceMsg.Media, tg.NewClient(primaryInv), sourceMsg) + if err != nil { + t.Fatal(err) + } + ctx := tgutil.ExtWithContext(context.Background(), primary) + statePath := ResumeStatePath(t.TempDir() + "/cache") + + first, firstCleanup := PrepareResumable(ctx, file, statePath) + if first == nil || firstCleanup == nil { + t.Fatal("relay was not prepared") + } + defer firstCleanup(ctx) + primaryInv.mu.Lock() + forwards := primaryInv.forwards + primaryInv.mu.Unlock() + if forwards != 1 { + t.Fatalf("forward count = %d, want 1", forwards) + } + if _, err := loadResumeState(statePath); err != nil { + t.Fatalf("state not persisted: %v", err) + } + + second, cleanup2 := PrepareResumable(ctx, file, statePath) + if second == nil || cleanup2 == nil { + t.Fatal("relay state was not reused") + } + primaryInv.mu.Lock() + forwards = primaryInv.forwards + primaryInv.mu.Unlock() + if forwards != 1 { + t.Fatalf("second prepare forwarded again: %d", forwards) + } + cleanup2(ctx) + if state, err := loadResumeState(statePath); err != nil || state != nil { + t.Fatalf("relay state after cleanup = %+v, %v", state, err) + } + primaryInv.mu.Lock() + deletes := primaryInv.deletes + primaryInv.mu.Unlock() + if deletes == 0 { + t.Fatal("relay message was not deleted") + } +} diff --git a/client/relay/state.go b/client/relay/state.go new file mode 100644 index 00000000..48c51ad7 --- /dev/null +++ b/client/relay/state.go @@ -0,0 +1,112 @@ +package relay + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/tfile" +) + +const resumeStateVersion = 1 + +type resumeState struct { + Version int `json:"version"` + RelayChatID int64 `json:"relay_chat_id"` + RelayMessageID int `json:"relay_message_id"` + SourceKind string `json:"source_kind"` + SourceID int64 `json:"source_id"` + SourceAccessHash int64 `json:"source_access_hash"` +} + +// ResumeStatePath returns the sidecar path used to keep a relay message alive +// across a crash. It deliberately lives next to the download cache. +func ResumeStatePath(cachePath string) string { + return cachePath + ".relay" +} + +func loadResumeState(path string) (*resumeState, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read relay resume state: %w", err) + } + var state resumeState + if err := json.Unmarshal(data, &state); err != nil { + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, fmt.Errorf("remove invalid relay resume state: %w", removeErr) + } + return nil, nil + } + if state.Version != resumeStateVersion || state.RelayMessageID == 0 || state.RelayChatID == 0 { + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, fmt.Errorf("remove incompatible relay resume state: %w", removeErr) + } + return nil, nil + } + return &state, nil +} + +func saveResumeState(path string, state resumeState) error { + data, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("marshal relay resume state: %w", err) + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("write relay resume state: %w", err) + } + return os.Rename(tmp, path) +} + +func removeResumeState(path string) error { + if path == "" { + return nil + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.Remove(path + ".tmp"); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +// CleanupResumeState releases a persisted relay message after the cache was +// already complete, for example if the process stopped between download and +// cleanup. +func CleanupResumeState(ctx context.Context, statePath string) { + state, err := loadResumeState(statePath) + if err != nil || state == nil { + return + } + extCtx := tgutil.ExtFromContext(ctx) + if extCtx == nil { + if err := removeResumeState(statePath); err != nil { + log.FromContext(ctx).Warnf("Failed to remove relay resume state: %v", err) + } + return + } + relayed := &relayFile{ + primary: extCtx, + relayChatID: state.RelayChatID, + relayMessageID: state.RelayMessageID, + keepMessages: config.C().Telegram.RelayKeepMessages, + statePath: statePath, + } + relayed.cleanup(ctx) +} + +func (s *resumeState) matches(identity tfile.FileIdentity) bool { + if s == nil { + return false + } + return s.SourceKind == identity.Kind && s.SourceID == identity.ID && s.SourceAccessHash == identity.AccessHash +} diff --git a/client/relay/state_test.go b/client/relay/state_test.go new file mode 100644 index 00000000..4d1bb552 --- /dev/null +++ b/client/relay/state_test.go @@ -0,0 +1,129 @@ +package relay + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/gotd/td/bin" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/worker" + "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/ext" + tgstorage "github.com/krau/mygotg/storage" +) + +type fakeInvoker struct { + mu sync.Mutex + calls map[uint]int + fail map[uint]bool + data []byte +} + +func (f *fakeInvoker) Invoke(_ context.Context, input bin.Encoder, output bin.Decoder) error { + req, ok := input.(*tg.UploadGetFileRequest) + if !ok { + return errors.New("unexpected request") + } + id := req.Location.(*tg.InputDocumentFileLocation).ID + f.mu.Lock() + f.calls[uint(id)]++ + fail := f.fail[uint(id)] + f.mu.Unlock() + if fail { + return errors.New("mock worker failure") + } + box, ok := output.(*tg.UploadFileBox) + if !ok { + return errors.New("unexpected response") + } + offset := int(req.Offset) + end := offset + req.Limit + if end > len(f.data) { + end = len(f.data) + } + box.File = &tg.UploadFile{Bytes: append([]byte(nil), f.data[offset:end]...)} + return nil +} + +func testExt(t *testing.T, inv *fakeInvoker) *ext.Context { + t.Helper() + peers := tgstorage.NewPeerStorage(nil, true) + peers.AddPeer(500, 1, tgstorage.TypeChannel, "") + return ext.NewContext(context.Background(), tg.NewClient(inv), peers, &tg.User{}, nil, nil, false) +} + +func TestResumeStateRoundTrip(t *testing.T) { + path := ResumeStatePath(t.TempDir() + "/cache.bin") + identity := tfile.FileIdentity{Kind: "document", ID: 7, AccessHash: 8} + state := resumeState{ + Version: resumeStateVersion, RelayChatID: 500, RelayMessageID: 42, + SourceKind: identity.Kind, SourceID: identity.ID, SourceAccessHash: identity.AccessHash, + } + if err := saveResumeState(path, state); err != nil { + t.Fatal(err) + } + loaded, err := loadResumeState(path) + if err != nil { + t.Fatal(err) + } + if loaded == nil || loaded.RelayMessageID != 42 || !loaded.matches(identity) { + t.Fatalf("loaded state = %+v", loaded) + } + if loaded.matches(tfile.FileIdentity{Kind: "document", ID: 9, AccessHash: 8}) { + t.Fatal("state matched a different file") + } + if err := removeResumeState(path); err != nil { + t.Fatal(err) + } + if loaded, err := loadResumeState(path); err != nil || loaded != nil { + t.Fatalf("state after removal = %+v, %v", loaded, err) + } +} + +func TestMultiClientFailsOverAndFallsBackToPrimary(t *testing.T) { + ctx := context.Background() + data := []byte("relay chunk") + firstInv := &fakeInvoker{calls: map[uint]int{}, fail: map[uint]bool{1: true}, data: data} + secondInv := &fakeInvoker{calls: map[uint]int{}, fail: map[uint]bool{}, data: data} + primaryInv := &fakeInvoker{calls: map[uint]int{}, fail: map[uint]bool{}, data: data} + location := &tg.InputDocumentFileLocation{ID: 1} + file := &relayFile{ + workers: []worker.ClientRef{ + {ID: 1, Name: "one", Ext: testExt(t, firstInv)}, + {ID: 2, Name: "two", Ext: testExt(t, secondInv)}, + }, + locations: map[uint]tg.InputFileLocationClass{ + 1: &tg.InputDocumentFileLocation{ID: 1}, + 2: &tg.InputDocumentFileLocation{ID: 2}, + }, + served: make(map[uint]int), + } + client := &multiClient{file: file, base: tg.NewClient(primaryInv)} + res, err := client.UploadGetFile(ctx, &tg.UploadGetFileRequest{Location: location, Offset: 0, Limit: len(data)}) + if err != nil { + t.Fatal(err) + } + uploaded, ok := res.(*tg.UploadFile) + if !ok || string(uploaded.Bytes) != string(data) { + t.Fatalf("response = %#v", res) + } + if firstInv.calls[1] != 1 || secondInv.calls[2] != 1 { + t.Fatalf("worker calls: first=%v second=%v", firstInv.calls, secondInv.calls) + } + + // Both helpers fail; the primary client must serve the chunk. + firstInv.fail[1] = true + secondInv.fail[2] = true + res, err = client.UploadGetFile(ctx, &tg.UploadGetFileRequest{Location: location, Offset: 0, Limit: len(data)}) + if err != nil { + t.Fatal(err) + } + if uploads, ok := res.(*tg.UploadFile); !ok || string(uploads.Bytes) != string(data) { + t.Fatalf("fallback response = %#v", res) + } + if primaryInv.calls[1] != 1 { + t.Fatalf("primary calls = %v", primaryInv.calls) + } +} diff --git a/client/user/auth_terminal.go b/client/user/auth_terminal.go deleted file mode 100644 index 96811cbc..00000000 --- a/client/user/auth_terminal.go +++ /dev/null @@ -1,57 +0,0 @@ -package user - -import ( - "bufio" - "fmt" - "os" - "strings" - - "github.com/celestix/gotgproto" - "golang.org/x/term" -) - -type terminalAuthConversator struct{} - -func readLine(prompt string) (string, error) { - fmt.Print(prompt) - reader := bufio.NewReader(os.Stdin) - text, err := reader.ReadString('\n') - if err != nil { - return "", err - } - return strings.TrimSpace(text), nil -} - -func (t *terminalAuthConversator) AskPhoneNumber() (string, error) { - fmt.Println("Your Phone Number (e.g. +44 123456):") - return readLine("> ") -} - -func (t *terminalAuthConversator) AskCode() (string, error) { - fmt.Println("Your Code (e.g. 123456):") - return readLine("> ") -} - -func (t *terminalAuthConversator) AskPassword() (string, error) { - fmt.Println("Your 2FA Password:") - fmt.Print("> ") - bytePwd, err := term.ReadPassword(int(os.Stdin.Fd())) - fmt.Println() - if err != nil { - return "", err - } - - return strings.TrimSpace(string(bytePwd)), nil -} - -func (t *terminalAuthConversator) AuthStatus(authStatus gotgproto.AuthStatus) { - switch authStatus.Event { - case gotgproto.AuthStatusPhoneRetrial: - fmt.Printf("The phone number is incorrect. Attempts left: %d\n", authStatus.AttemptsLeft) - case gotgproto.AuthStatusPasswordRetrial: - fmt.Printf("The 2FA password is incorrect. Attempts left: %d\n", authStatus.AttemptsLeft) - case gotgproto.AuthStatusPhoneCodeRetrial: - fmt.Printf("The OTP code is incorrect. Attempts left: %d\n", authStatus.AttemptsLeft) - default: - } -} diff --git a/client/user/sync.go b/client/user/sync.go new file mode 100644 index 00000000..bc347300 --- /dev/null +++ b/client/user/sync.go @@ -0,0 +1,52 @@ +package user + +import ( + "context" + "errors" + "sync" + + "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/query/dialogs" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/mygotg/storage" +) + +var syncPeersMu sync.Mutex + +// SyncPeers imports the userbot's dialogs into the shared peer cache. +func SyncPeers(ctx context.Context) (int, error) { + if !config.C().Telegram.Userbot.Enable { + return 0, errors.New("userbot is not enabled") + } + syncPeersMu.Lock() + defer syncPeersMu.Unlock() + uctx := GetCtx() + if uctx == nil { + return 0, errors.New("userbot is not running") + } + tapi := uctx.Raw + peerStorage := uctx.PeerStorage + log.FromContext(ctx).Info("Starting to sync peers...") + count := 0 + err := dialogs.NewQueryBuilder(tapi).GetDialogs().BatchSize(50).ForEach(ctx, func(ctx context.Context, e dialogs.Elem) error { + for cid, channel := range e.Entities.Channels() { + peerStorage.AddPeer(cid, channel.AccessHash, storage.TypeChannel, channel.Username) + count++ + } + for uid, user := range e.Entities.Users() { + peerStorage.AddPeer(uid, user.AccessHash, storage.TypeUser, user.Username) + count++ + } + for gid := range e.Entities.Chats() { + peerStorage.AddPeer(gid, storage.DefaultAccessHash, storage.TypeChat, storage.DefaultUsername) + count++ + } + return nil + }) + if err != nil { + log.FromContext(ctx).Error("Failed to sync peers", "error", err) + return count, err + } + log.FromContext(ctx).Info("Finished syncing peers") + return count, nil +} diff --git a/client/user/userclient.go b/client/user/userclient.go index edc9dbc6..774e3224 100644 --- a/client/user/userclient.go +++ b/client/user/userclient.go @@ -2,113 +2,264 @@ package user import ( "context" + "errors" + "fmt" + "os" + "runtime/debug" "sync" "time" - "github.com/celestix/gotgproto" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/dispatcher/handlers" - "github.com/celestix/gotgproto/dispatcher/handlers/filters" - "github.com/celestix/gotgproto/ext" - "github.com/celestix/gotgproto/sessionMaker" - "github.com/charmbracelet/log" "github.com/gotd/td/tg" + gotgproto "github.com/krau/mygotg" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/dispatcher/handlers" + "github.com/krau/mygotg/dispatcher/handlers/filters" + gotgerrors "github.com/krau/mygotg/errors" + "github.com/krau/mygotg/ext" + "github.com/krau/mygotg/session" + "github.com/krau/SaveAny-Bot/client/middleware" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/database" ) -var uc *gotgproto.Client +var ErrLoginRequired = errors.New("telegram login required") + +type Status struct { + State string + ID int64 + Username string + Phone string + Error string +} + +type failConversator struct{} + +func (failConversator) AskPhoneNumber() (string, error) { return "", ErrLoginRequired } +func (failConversator) AskCode() (string, error) { return "", ErrLoginRequired } +func (failConversator) AskPassword() (string, error) { return "", ErrLoginRequired } +func (failConversator) AuthStatus(gotgproto.AuthStatus) {} -// getEctx lazily creates the user-client ext.Context exactly once. Guarded by -// sync.OnceValue so concurrent GetCtx calls cannot race on ectx creation. -var getEctx = sync.OnceValue(func() *ext.Context { - return uc.CreateContext() -}) +var ( + ucMu sync.RWMutex + uc *gotgproto.Client + uctx *ext.Context + ustatus = Status{State: "stopped"} + starting bool + generation uint64 + startCancel context.CancelFunc +) + +func CurrentStatus() Status { + ucMu.RLock() + defer ucMu.RUnlock() + return ustatus +} +// GetCtx returns the userbot context, or nil when the userbot is not running. func GetCtx() *ext.Context { - if uc == nil { - return nil - } - return getEctx() + ucMu.RLock() + defer ucMu.RUnlock() + return uctx } -func Login(ctx context.Context) (*gotgproto.Client, error) { - log.FromContext(ctx).Debug("Logging in user client") +// Start connects the userbot, asking conv for credentials when the session is +// not authorized. Passing nil uses a conversator that fails with +// ErrLoginRequired. +func Start(ctx context.Context, conv gotgproto.AuthConversator, phone string) error { + ucMu.Lock() if uc != nil { - return uc, nil + ucMu.Unlock() + return nil } - res := make(chan struct { + if starting { + ucMu.Unlock() + return errors.New("userbot is already starting") + } + starting = true + generation++ + gen := generation + startCtx, cancelStart := context.WithCancel(ctx) + startCancel = cancelStart + ustatus = Status{State: "starting"} + ucMu.Unlock() + + autoStart := conv == nil + if conv == nil { + conv = failConversator{} + } + logger := log.FromContext(ctx) + logger.Debug("Logging in user client") + + type result struct { client *gotgproto.Client err error - }) + } + res := make(chan result, 1) go func() { + sent := false + defer func() { + if r := recover(); r != nil { + logger.Errorf("Userbot client initialization panicked: %v\n%s", r, debug.Stack()) + if !sent { + res <- result{nil, fmt.Errorf("userbot client initialization panicked: %v", r)} + } + } + }() resolver, err := tgutil.NewConfigProxyResolver() if err != nil { - res <- struct { - client *gotgproto.Client - err error - }{nil, err} + res <- result{nil, err} + sent = true return } tclient, err := gotgproto.NewClient( config.C().Telegram.AppID, config.C().Telegram.AppHash, - gotgproto.ClientTypePhone(""), + gotgproto.ClientTypePhone(phone), &gotgproto.ClientOpts{ - Session: sessionMaker.SqlSession(database.GetDialect(config.C().Telegram.Userbot.Session)), - AuthConversator: &terminalAuthConversator{}, - Context: ctx, - DisableCopyright: true, - Resolver: resolver, - MaxRetries: config.C().Telegram.RpcRetry, - AutoFetchReply: true, - Middlewares: middleware.NewDefaultMiddlewares(ctx, 5*time.Minute), + Session: session.SqlSession(database.GetDialect(config.C().Telegram.Userbot.Session)), + AuthConversator: conv, + NoAutoAuth: autoStart, + Context: startCtx, + DisableCopyright: true, + DeferUpdateRecovery: true, + Resolver: resolver, + MaxRetries: config.C().Telegram.RpcRetry, + AutoFetchReply: true, + Middlewares: middleware.NewDefaultMiddlewares(startCtx, 5*time.Minute), ErrorHandler: func(ctx *ext.Context, u *ext.Update, s string) error { log.FromContext(ctx).Errorf("Unhandled error: %s", s) return dispatcher.EndGroups }, }, ) - if err != nil { - res <- struct { - client *gotgproto.Client - err error - }{nil, err} - } - res <- struct { - client *gotgproto.Client - err error - }(struct { - client *gotgproto.Client - err error - }{tclient, nil}) + res <- result{tclient, err} + sent = true }() + var r result select { - case <-ctx.Done(): - return nil, ctx.Err() - case r := <-res: - if r.err != nil { - return nil, r.err + case <-startCtx.Done(): + ucMu.Lock() + if generation == gen { + starting = false + generation++ + startCancel = nil + ustatus = Status{State: "stopped"} + } + ucMu.Unlock() + return startCtx.Err() + case r = <-res: + } + if r.err != nil { + state := "error" + if errors.Is(r.err, ErrLoginRequired) || (autoStart && errors.Is(r.err, gotgerrors.ErrSessionUnauthorized)) { + state = "login_required" + } + ucMu.Lock() + if generation == gen { + starting = false + startCancel = nil + ustatus = Status{State: state, Error: r.err.Error(), Phone: phone} + } + ucMu.Unlock() + return r.err + } + + if startCtx.Err() != nil { + r.client.Stop() + ucMu.Lock() + if generation == gen { + starting = false + startCancel = nil + ustatus = Status{State: "stopped"} + } + ucMu.Unlock() + return startCtx.Err() + } + tclient := r.client + tclient.Dispatcher.AddHandler(handlers.NewMessage(filters.Message.Media, func(ctx *ext.Context, u *ext.Update) error { + switch u.UpdateClass.(type) { + case *tg.UpdateEditChannelMessage, *tg.UpdateEditMessage, *tg.UpdateDeleteChannelMessages, *tg.UpdateDeleteMessages: + return dispatcher.EndGroups + } + chatID := tgutil.ChatIdFromPeer(u.EffectiveMessage.GetPeerID()) + watchChats, err := database.GetWatchChatsByChatID(ctx, chatID) + if err != nil || len(watchChats) == 0 { + return dispatcher.EndGroups + } + return dispatcher.ContinueGroups + })) + tclient.Dispatcher.AddHandler(handlers.NewMessage(filters.Message.Media, handleMediaMessage)) + if err := tclient.StartUpdateRecovery(startCtx); err != nil { + tclient.Stop() + ucMu.Lock() + if generation == gen { + starting = false + startCancel = nil + ustatus = Status{State: "error", Error: err.Error(), Phone: phone} + } + ucMu.Unlock() + return err + } + + status := Status{State: "running", Phone: phone} + if tclient.Self != nil { + status.ID = tclient.Self.ID + status.Username = tclient.Self.Username + } + ucMu.Lock() + if generation != gen { + ucMu.Unlock() + tclient.Stop() + return errors.New("userbot start canceled") + } + uc = tclient + uctx = tclient.CreateContext() + starting = false + startCancel = nil + ustatus = status + ucMu.Unlock() + + if tclient.Self != nil { + logger.Infof("User client logged in successfully: %s %s", tclient.Self.FirstName, tclient.Self.LastName) + } else { + logger.Info("User client logged in successfully") + } + return nil +} + +func Stop() { + ucMu.Lock() + client := uc + uc = nil + uctx = nil + starting = false + if startCancel != nil { + startCancel() + startCancel = nil + } + generation++ + ustatus = Status{State: "stopped"} + ucMu.Unlock() + if client != nil { + client.Stop() + } +} + +// Logout stops the userbot and removes its persisted session. +func Logout() error { + Stop() + sessionPath := config.C().Telegram.Userbot.Session + if sessionPath == "" { + return nil + } + for _, path := range []string{sessionPath, sessionPath + "-wal", sessionPath + "-shm"} { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err } - uc = r.client - uc.Dispatcher.AddHandler(handlers.NewMessage(filters.Message.Media, func(ctx *ext.Context, u *ext.Update) error { - switch u.UpdateClass.(type) { - case *tg.UpdateEditChannelMessage, *tg.UpdateEditMessage, *tg.UpdateDeleteChannelMessages, *tg.UpdateDeleteMessages: - return dispatcher.EndGroups - } - chatId := u.EffectiveChat().GetID() - watchChats, err := database.GetWatchChatsByChatID(ctx, chatId) - if err != nil || len(watchChats) == 0 { - return dispatcher.EndGroups - } - return dispatcher.ContinueGroups - })) - uc.Dispatcher.AddHandler(handlers.NewMessage(filters.Message.Media, handleMediaMessage)) - log.FromContext(ctx).Infof("User client logged in successfully: %s", uc.Self.FirstName+" "+uc.Self.LastName) - return uc, nil } + return nil } diff --git a/client/user/watch.go b/client/user/watch.go index 5506fb39..94b65d48 100644 --- a/client/user/watch.go +++ b/client/user/watch.go @@ -4,11 +4,11 @@ import ( "sync" "time" - "github.com/celestix/gotgproto/dispatcher" - "github.com/celestix/gotgproto/ext" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/mygotg/dispatcher" + "github.com/krau/mygotg/ext" ) type MediaMessageEvent struct { @@ -51,16 +51,18 @@ func sendMediaMessageEvent(event MediaMessageEvent) { if timer, exists := mediaMessageHandler.timers[key]; exists { timer.Stop() - } else { - mediaMessageHandler.events[key] = event } + mediaMessageHandler.events[key] = event mediaMessageHandler.timers[key] = time.AfterFunc(mediaMessageHandler.debounce, func() { mediaMessageHandler.mu.Lock() - event := mediaMessageHandler.events[key] + event, ok := mediaMessageHandler.events[key] delete(mediaMessageHandler.events, key) delete(mediaMessageHandler.timers, key) mediaMessageHandler.mu.Unlock() + if !ok { + return + } mediaMessageCh <- event }) @@ -89,7 +91,7 @@ func handleMediaMessage(ctx *ext.Context, update *ext.Update) error { if err != nil { return err } - chatId := update.EffectiveChat().GetID() + chatId := tgutil.ChatIdFromPeer(update.EffectiveMessage.GetPeerID()) sendMediaMessageEvent(MediaMessageEvent{ Ctx: ctx, ChatID: chatId, diff --git a/client/worker/manager.go b/client/worker/manager.go new file mode 100644 index 00000000..7f3e621f --- /dev/null +++ b/client/worker/manager.go @@ -0,0 +1,375 @@ +package worker + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + + "github.com/charmbracelet/log" + "github.com/krau/mygotg" + "github.com/krau/mygotg/ext" + "github.com/krau/mygotg/session" + "github.com/rs/xid" + + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/database" +) + +const ( + StateStopped = "stopped" + StateStarting = "starting" + StateRunning = "running" + StateError = "error" +) + +type Status struct { + State string `json:"state"` + Username string `json:"username,omitempty"` + Error string `json:"error,omitempty"` +} + +type ClientRef struct { + ID uint + Name string + Ext *ext.Context +} + +type runtime struct { + id uint + name string + session string + mu sync.RWMutex + state string + err string + username string + client *mygotg.Client + ext *ext.Context + cancel context.CancelFunc + gen uint64 +} + +type Manager struct { + ctx context.Context + dataDir string + mu sync.RWMutex + workers map[uint]*runtime + order []uint + next int +} + +var manager *Manager + +// Init creates the optional helper-bot manager. A manager with no workers is a +// no-op, preserving the zero-config single-bot setup. +func Init(ctx context.Context, dataDir string) { + m := &Manager{ctx: ctx, dataDir: dataDir, workers: make(map[uint]*runtime)} + manager = m + if err := m.reload(ctx); err != nil { + log.Errorf("Failed to load helper bots: %v", err) + } +} + +func (m *Manager) reload(ctx context.Context) error { + rows, err := database.ListBotWorkers(ctx) + if err != nil { + return err + } + m.mu.Lock() + old := m.workers + m.workers = make(map[uint]*runtime, len(rows)) + m.order = m.order[:0] + m.next = 0 + m.mu.Unlock() + for _, rt := range old { + stopRuntime(rt) + } + for _, row := range rows { + m.mu.Lock() + rt := &runtime{id: row.ID, name: row.Name, session: row.SessionPath, state: StateStopped} + m.workers[row.ID] = rt + m.order = append(m.order, row.ID) + m.mu.Unlock() + if row.Enabled { + m.start(ctx, rt, row.Token) + } + } + return nil +} + +func (m *Manager) start(ctx context.Context, rt *runtime, token string) { + if token == "" { + rt.mu.Lock() + rt.state, rt.err = StateError, "token is empty" + rt.mu.Unlock() + return + } + sessionPath := rt.session + if sessionPath == "" { + sessionPath = NewSessionPath(m.dataDir) + if err := database.UpdateBotWorker(ctx, rt.id, map[string]any{"session_path": sessionPath}); err != nil { + rt.mu.Lock() + rt.state, rt.err = StateError, err.Error() + rt.mu.Unlock() + return + } + } + if err := os.MkdirAll(filepath.Dir(sessionPath), 0o755); err != nil { + rt.mu.Lock() + rt.state, rt.err = StateError, err.Error() + rt.mu.Unlock() + return + } + + runCtx, cancel := context.WithCancel(ctx) + rt.mu.Lock() + rt.session = sessionPath + rt.gen++ + gen := rt.gen + rt.cancel = cancel + rt.state, rt.err, rt.username = StateStarting, "", "" + rt.mu.Unlock() + + go func() { + resolver, err := tgutil.NewConfigProxyResolver() + if err != nil { + m.finishStart(rt, gen, cancel, nil, err, "") + return + } + client, err := mygotg.NewClient( + config.C().Telegram.AppID, + config.C().Telegram.AppHash, + mygotg.ClientTypeBot(token), + &mygotg.ClientOpts{ + Session: session.SqlSession(database.GetDialect(sessionPath)), + Context: runCtx, + NoUpdates: true, + DisableCopyright: true, + Resolver: resolver, + MaxRetries: config.C().Telegram.RpcRetry, + }, + ) + if err != nil { + m.finishStart(rt, gen, cancel, nil, err, "") + return + } + username := "" + if client.Self != nil { + username = client.Self.Username + } + m.finishStart(rt, gen, cancel, client, nil, username) + }() +} + +func (m *Manager) finishStart(rt *runtime, gen uint64, cancel context.CancelFunc, client *mygotg.Client, err error, username string) { + rt.mu.Lock() + defer rt.mu.Unlock() + if rt.gen != gen { + if client != nil { + client.Stop() + } + cancel() + return + } + if err != nil { + rt.state, rt.err = StateError, err.Error() + rt.cancel = nil + cancel() + return + } + rt.client = client + rt.ext = client.CreateContext() + rt.state = StateRunning + rt.err = "" + rt.username = username +} + +// StartWorker starts one persisted worker and returns its current status. +func StartWorker(ctx context.Context, id uint) (Status, error) { + if manager == nil { + return Status{}, errors.New("helper bot manager is not initialized") + } + row, err := database.GetBotWorker(ctx, id) + if err != nil { + return Status{}, err + } + if !row.Enabled { + return Status{}, errors.New("helper bot is disabled") + } + manager.mu.Lock() + rt := manager.workers[id] + if rt == nil { + rt = &runtime{id: row.ID, name: row.Name, session: row.SessionPath, state: StateStopped} + manager.workers[id] = rt + manager.order = append(manager.order, id) + } + manager.mu.Unlock() + rt.mu.RLock() + state := rt.state + rt.mu.RUnlock() + if state == StateRunning || state == StateStarting { + return StatusFor(id), nil + } + rt.mu.Lock() + rt.name = row.Name + if row.SessionPath != "" { + rt.session = row.SessionPath + } + rt.mu.Unlock() + lifecycleCtx := manager.ctx + if lifecycleCtx == nil { + lifecycleCtx = ctx + } + manager.start(lifecycleCtx, rt, row.Token) + return StatusFor(id), nil +} + +// StopWorker stops one worker while keeping its persisted row. +func StopWorker(id uint) { + if manager == nil { + return + } + manager.mu.RLock() + rt := manager.workers[id] + manager.mu.RUnlock() + if rt != nil { + stopRuntime(rt) + } +} + +func StopAll() { + if manager == nil { + return + } + manager.mu.RLock() + workers := make([]*runtime, 0, len(manager.workers)) + for _, rt := range manager.workers { + workers = append(workers, rt) + } + manager.mu.RUnlock() + for _, rt := range workers { + stopRuntime(rt) + } +} + +func stopRuntime(rt *runtime) { + rt.mu.Lock() + client := rt.client + rt.client = nil + rt.ext = nil + rt.state, rt.err, rt.username = StateStopped, "", "" + rt.gen++ + cancel := rt.cancel + rt.cancel = nil + rt.mu.Unlock() + if cancel != nil { + cancel() + } + if client != nil { + client.Stop() + } +} + +// UploadExts returns running helper contexts in round-robin order. Callers can +// skip helpers that cannot access the intended target chat. +func UploadExts() []*ext.Context { + if manager == nil { + return nil + } + manager.mu.Lock() + defer manager.mu.Unlock() + if len(manager.order) == 0 { + return nil + } + exts := make([]*ext.Context, 0, len(manager.order)) + start := manager.next + first := -1 + for i := range manager.order { + idx := (start + i) % len(manager.order) + rt := manager.workers[manager.order[idx]] + if rt == nil { + continue + } + rt.mu.RLock() + state, extCtx := rt.state, rt.ext + rt.mu.RUnlock() + if state == StateRunning && extCtx != nil { + if first == -1 { + first = idx + } + exts = append(exts, extCtx) + } + } + if first >= 0 { + manager.next = (first + 1) % len(manager.order) + } + return exts +} + +// UploadExt returns the next running helper context for one Telegram upload, +// or nil when no helper is configured. +func UploadExt() *ext.Context { + exts := UploadExts() + if len(exts) > 0 { + return exts[0] + } + return nil +} + +// RunningClients returns a snapshot of running helpers for download routing. +func RunningClients() []ClientRef { + if manager == nil { + return nil + } + manager.mu.RLock() + defer manager.mu.RUnlock() + refs := make([]ClientRef, 0, len(manager.order)) + for _, id := range manager.order { + rt := manager.workers[id] + if rt == nil { + continue + } + rt.mu.RLock() + if rt.state == StateRunning && rt.ext != nil { + refs = append(refs, ClientRef{ID: rt.id, Name: rt.name, Ext: rt.ext}) + } + rt.mu.RUnlock() + } + return refs +} + +// StatusFor returns the runtime status for a worker id. +func StatusFor(id uint) Status { + if manager == nil { + return Status{State: StateStopped} + } + manager.mu.RLock() + rt := manager.workers[id] + manager.mu.RUnlock() + if rt == nil { + return Status{State: StateStopped} + } + rt.mu.RLock() + defer rt.mu.RUnlock() + return Status{State: rt.state, Username: rt.username, Error: rt.err} +} + +// NewSessionPath returns a unique session database path under data/workers. +func NewSessionPath(dataDir string) string { + return filepath.Join(dataDir, "workers", xid.New().String()+".db") +} + +// RemoveSession deletes a worker session database and its SQLite side files. +func RemoveSession(path string) error { + if path == "" { + return nil + } + for _, suffix := range []string{"", "-wal", "-shm"} { + if err := os.Remove(path + suffix); err != nil && !os.IsNotExist(err) { + return err + } + } + return nil +} diff --git a/cmd/root.go b/cmd/root.go index 35bb6444..de99aad0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,11 +2,12 @@ package cmd import ( "context" - "fmt" + "os" + "github.com/krau/SaveAny-Bot/boot" + "github.com/krau/SaveAny-Bot/cmd/testenv" "github.com/krau/SaveAny-Bot/cmd/upload" "github.com/krau/SaveAny-Bot/cmd/watch" - "github.com/krau/SaveAny-Bot/config" "github.com/spf13/cobra" ) @@ -17,13 +18,14 @@ var rootCmd = &cobra.Command{ } func init() { - config.RegisterFlags(rootCmd) + boot.RegisterFlags(rootCmd) + testenv.Register(rootCmd) upload.Register(rootCmd) watch.Register(rootCmd) } func Execute(ctx context.Context) { if err := rootCmd.ExecuteContext(ctx); err != nil { - fmt.Println(err) + os.Exit(1) } } diff --git a/cmd/run.go b/cmd/run.go index cf244567..392abb46 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -7,26 +7,20 @@ import ( "strings" "time" - "slices" - "github.com/charmbracelet/log" - "github.com/krau/SaveAny-Bot/api" - "github.com/krau/SaveAny-Bot/client/bot" - userclient "github.com/krau/SaveAny-Bot/client/user" - "github.com/krau/SaveAny-Bot/common/cache" - "github.com/krau/SaveAny-Bot/common/i18n" + "github.com/spf13/cobra" + + "github.com/krau/SaveAny-Bot/app" + "github.com/krau/SaveAny-Bot/boot" "github.com/krau/SaveAny-Bot/common/selfupdate" "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/config" - "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/database" - "github.com/krau/SaveAny-Bot/parsers" - "github.com/krau/SaveAny-Bot/storage" - "github.com/spf13/cobra" ) func Run(cmd *cobra.Command, _ []string) { ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() logger := log.NewWithOptions(os.Stdout, log.Options{ Level: log.InfoLevel, ReportTimestamp: true, @@ -38,34 +32,36 @@ func Run(cmd *cobra.Command, _ []string) { selfupdate.CleanLeftovers() - configFile := config.GetConfigFile(cmd) - if err := config.Init(ctx, configFile); err != nil { + opts := boot.Parse(cmd) + opts.InitWeb = true + res, err := boot.Init(ctx, opts) + if err != nil { logger.Fatal("Init failed", "error", err) } - - level, err := log.ParseLevel(strings.TrimSpace(config.C().Log.Level)) - if err != nil { - logger.Warn("Invalid log level, fallback to debug", "level", config.C().Log.Level, "error", err) - level = log.DebugLevel + cfg := res.Config + if level, err := log.ParseLevel(strings.TrimSpace(cfg.Log.Level)); err == nil { + logger.SetLevel(level) } - logger.SetLevel(level) - exitChan, err := initAll(ctx) - if err != nil { + if unfinished, err := database.HasUnfinishedTasks(ctx); err != nil { + logger.Errorf("Failed to inspect unfinished tasks; keeping cache: %v", err) + } else if !unfinished { + cleanCache(cfg) + } + application := app.New(ctx, res.Settings, app.Options{ + DataDir: res.DataDir, + DBPath: res.DBPath, + Listen: res.Listen, + NoWeb: res.NoWeb, + }) + if err := application.Start(); err != nil { logger.Fatal("Init failed", "error", err) } - go func() { - <-exitChan - cancel() - }() - core.Run(ctx) - - <-ctx.Done() + <-application.Done() logger.Info("Exiting...") defer logger.Info("Exit complete") - core.Close() - cleanCache() + application.Stop() if selfupdate.ShouldRestart() { logger.Info("Restarting to run the updated version...") if err := selfupdate.Restart(); err != nil { @@ -74,54 +70,34 @@ func Run(cmd *cobra.Command, _ []string) { } } -func initAll(ctx context.Context) (<-chan struct{}, error) { - cache.Init() - logger := log.FromContext(ctx) - i18n.Init(config.C().Lang) - logger.Info("Initializing...") - database.Init(ctx) - storage.LoadStorages(ctx) - if config.C().Parser.PluginEnable { - for _, dir := range config.C().Parser.PluginDirs { - if err := parsers.LoadPlugins(ctx, dir); err != nil { - logger.Error("Failed to load parser plugins", "dir", dir, "error", err) - } else { - logger.Debug("Loaded parser plugins from directory", "dir", dir) - } - } - } - if config.C().Telegram.Userbot.Enable { - _, err := userclient.Login(ctx) - if err != nil { - logger.Fatal("User login failed", "error", err) - } - } - if err := api.Start(ctx); err != nil { - logger.Fatal("Failed to start API server", "error", err) - } - return bot.Init(ctx), nil -} - -func cleanCache() { - if config.C().NoCleanCache { +func cleanCache(cfg config.Config) { + if cfg.NoCleanCache { return } - if config.C().Temp.BasePath != "" && !config.C().Stream { - if slices.Contains([]string{"/", ".", "\\", ".."}, filepath.Clean(config.C().Temp.BasePath)) { - log.Error("Invalid cache directory", "path", config.C().Temp.BasePath) - return - } + if cfg.Temp.BasePath != "" && !cfg.Stream { currentDir, err := os.Getwd() if err != nil { log.Error("Failed to get working directory", "error", err) return } - cachePath := filepath.Join(currentDir, config.C().Temp.BasePath) + cachePath := cfg.Temp.BasePath + if !filepath.IsAbs(cachePath) { + cachePath = filepath.Join(currentDir, cachePath) + } cachePath, err = filepath.Abs(cachePath) if err != nil { log.Error("Failed to get absolute cache path", "error", err) return } + cachePath = filepath.Clean(cachePath) + // Never clean the working directory or one of its ancestors. + if rel, err := filepath.Rel(cachePath, currentDir); err != nil || rel == "." || !strings.HasPrefix(rel, "..") { + log.Error("Refusing to clean unsafe cache directory", "path", cachePath) + return + } + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + return + } log.Info("Cleaning cache directory", "path", cachePath) if err := fsutil.RemoveAllInDir(cachePath); err != nil { log.Error("Failed to clean cache directory", "error", err) diff --git a/cmd/testenv/cmd.go b/cmd/testenv/cmd.go new file mode 100644 index 00000000..1407bf71 --- /dev/null +++ b/cmd/testenv/cmd.go @@ -0,0 +1,225 @@ +package testenv + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/charmbracelet/log" + "github.com/spf13/cobra" + + "github.com/krau/SaveAny-Bot/app" + "github.com/krau/SaveAny-Bot/boot" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/settings" +) + +var testenvCmd = &cobra.Command{ + Use: "testenv", + Short: "run a seeded local environment for end-to-end testing", + Long: `Run the web UI and task pipeline against a disposable, seeded data directory. + +No Telegram credentials are needed. The command creates an admin account, a +local storage and a whitelisted user, then prints the login details and a +sample URL that can be used to create a direct-links task.`, + RunE: run, +} + +func Register(root *cobra.Command) { + flags := testenvCmd.Flags() + flags.String("admin-password", "", "web admin password (default saveany-test)") + flags.String("storage-name", "test-local", "name of the seeded local storage") + flags.Int64("user-chat-id", 10001, "chat ID of the seeded whitelisted user") + flags.Bool("keep", false, "keep the generated data directory after exit") + root.AddCommand(testenvCmd) +} + +func run(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + opts := boot.Parse(cmd) + opts.InitWeb = true + + autoDir := false + if !cmd.Flags().Changed("data-dir") && strings.TrimSpace(os.Getenv("SAVEANY_DATA_DIR")) == "" { + dir, err := os.MkdirTemp("", "saveany-testenv-") + if err != nil { + return fmt.Errorf("create test data directory: %w", err) + } + opts.DataDir = dir + opts.DBPath = filepath.Join(dir, "saveany.db") + autoDir = true + } + + keep, err := cmd.Flags().GetBool("keep") + if err != nil { + return err + } + if autoDir && !keep { + defer func() { + if err := os.RemoveAll(opts.DataDir); err != nil { + log.Warn("Failed to remove test data directory", "path", opts.DataDir, "error", err) + } + }() + } + + password, err := cmd.Flags().GetString("admin-password") + if err != nil { + return err + } + if password == "" { + password = strings.TrimSpace(os.Getenv("SAVEANY_ADMIN_PASSWORD")) + } + if password == "" { + password = "saveany-test" + } + if err := os.Setenv("SAVEANY_ADMIN_PASSWORD", password); err != nil { + return fmt.Errorf("set admin password: %w", err) + } + + res, err := boot.Init(ctx, opts) + if err != nil { + return fmt.Errorf("initialize test environment: %w", err) + } + + storageName, err := cmd.Flags().GetString("storage-name") + if err != nil { + return err + } + chatID, err := cmd.Flags().GetInt64("user-chat-id") + if err != nil { + return err + } + downloadsDir := filepath.Join(opts.DataDir, "downloads") + if err := os.MkdirAll(downloadsDir, 0o755); err != nil { + return fmt.Errorf("create downloads directory: %w", err) + } + record, err := res.Settings.CreateStorage(ctx, settings.StorageInput{ + Name: storageName, + Type: "local", + Enable: true, + Config: map[string]any{"base_path": downloadsDir}, + }) + if err != nil && !errors.Is(err, settings.ErrStorageExists) { + return fmt.Errorf("seed local storage: %w", err) + } + if record == nil { + records, err := database.ListStorageRecords(ctx) + if err != nil { + return fmt.Errorf("find seeded storage: %w", err) + } + for i := range records { + if records[i].Name == storageName { + record = &records[i] + break + } + } + if record == nil { + return fmt.Errorf("seeded storage %q not found", storageName) + } + } + + if _, err := database.GetUserByChatID(ctx, chatID); err != nil { + if !database.IsNotFound(err) { + return fmt.Errorf("find seeded user: %w", err) + } + if err := database.CreateUser(ctx, chatID); err != nil { + return fmt.Errorf("create seeded user: %w", err) + } + } + user, err := database.GetUserByChatID(ctx, chatID) + if err != nil { + return fmt.Errorf("load seeded user: %w", err) + } + user.Enabled = true + user.DefaultStorageID = record.ID + if err := database.UpdateUser(ctx, user); err != nil { + return fmt.Errorf("update seeded user: %w", err) + } + if err := res.Settings.ReloadConfig(ctx); err != nil { + return fmt.Errorf("reload settings: %w", err) + } + + sampleServer, sampleURL, err := serveSample(ctx, opts.DataDir) + if err != nil { + return err + } + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := sampleServer.Shutdown(shutdownCtx); err != nil { + log.Warnf("Failed to shut down sample server: %v", err) + } + }() + + application := app.New(ctx, res.Settings, app.Options{ + DataDir: opts.DataDir, + DBPath: opts.DBPath, + Listen: res.Listen, + NoWeb: res.NoWeb, + }) + if err := application.Start(); err != nil { + return fmt.Errorf("start test environment: %w", err) + } + defer application.Stop() + + fmt.Println("Test environment ready") + fmt.Printf(" Web UI: %s\n", webURL(res.Listen)) + fmt.Printf(" Password: %s\n", password) + fmt.Printf(" Data dir: %s\n", opts.DataDir) + storagePath := downloadsDir + if record.ConfigJSON != "" { + var cfg map[string]any + if err := json.Unmarshal([]byte(record.ConfigJSON), &cfg); err == nil { + if path, ok := cfg["base_path"].(string); ok && path != "" { + storagePath = path + } + } + } + fmt.Printf(" Storage: %s -> %s\n", record.Name, storagePath) + fmt.Printf(" Test user: %d\n", chatID) + fmt.Printf(" Sample URL: %s\n", sampleURL) + fmt.Println("Press Ctrl+C to stop.") + + <-ctx.Done() + log.Info("Stopping test environment") + return nil +} + +func serveSample(_ context.Context, dataDir string) (*http.Server, string, error) { + dir := filepath.Join(dataDir, "sample") + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, "", fmt.Errorf("create sample directory: %w", err) + } + if err := os.WriteFile(filepath.Join(dir, "sample.txt"), []byte("SaveAny-Bot end-to-end sample\n"), 0o644); err != nil { + return nil, "", fmt.Errorf("write sample file: %w", err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, "", fmt.Errorf("listen for sample server: %w", err) + } + server := &http.Server{Handler: http.FileServer(http.Dir(dir)), ReadHeaderTimeout: 5 * time.Second} + go func() { + if err := server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Error("Sample server error", "error", err) + } + }() + return server, "http://" + ln.Addr().String() + "/sample.txt", nil +} + +func webURL(listen string) string { + host, port, err := net.SplitHostPort(listen) + if err != nil { + return "http://" + listen + } + if host == "" || host == "0.0.0.0" || host == "::" { + host = "127.0.0.1" + } + return "http://" + net.JoinHostPort(host, port) + "/" +} diff --git a/cmd/upload/cmd.go b/cmd/upload/cmd.go index 9648c595..9add90d0 100644 --- a/cmd/upload/cmd.go +++ b/cmd/upload/cmd.go @@ -1,31 +1,28 @@ package upload import ( - "context" "fmt" - "io" "os" "path" "path/filepath" + "time" "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/boot" "github.com/krau/SaveAny-Bot/client/bot" - "github.com/krau/SaveAny-Bot/common/cache" - "github.com/krau/SaveAny-Bot/common/i18n" - "github.com/krau/SaveAny-Bot/common/utils/ioutil" + "github.com/krau/SaveAny-Bot/common/upload" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" - "github.com/krau/SaveAny-Bot/database" - "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" stortype "github.com/krau/SaveAny-Bot/pkg/enums/storage" "github.com/krau/SaveAny-Bot/storage" "github.com/spf13/cobra" ) var uploadCmd = &cobra.Command{ - Use: "upload", - Short: "upload local files to storage", - RunE: Upload, + Use: "upload", + Short: "upload local files to storage", + SilenceUsage: true, + RunE: Upload, } func Register(root *cobra.Command) { @@ -57,71 +54,63 @@ func Upload(cmd *cobra.Command, args []string) error { } ctx := cmd.Context() - log := log.FromContext(ctx) - configFile := config.GetConfigFile(cmd) - if err := config.Init(ctx, configFile); err != nil { + logger := log.FromContext(ctx) + opts := boot.Parse(cmd) + opts.InitWeb = false + if _, err := boot.Init(ctx, opts); err != nil { return fmt.Errorf("failed to load config: %w", err) } - i18n.Init(config.C().Lang) - cache.Init() - database.Init(ctx) stor, err := storage.GetStorageByName(ctx, storname) if err != nil { - log.Fatal("Failed to get storage", "error", err) + return fmt.Errorf("failed to get storage %q: %w", storname, err) } switch stor.Type() { case stortype.Telegram: - bot.Init(ctx) - default: - // placeholder for other storage types that may need special initialization + if err := bot.Start(ctx); err != nil { + return fmt.Errorf("failed to start bot: %w", err) + } + defer bot.Stop() } - file, err := os.Open(filepath.Clean(fp)) + fileInfo, err := os.Stat(filepath.Clean(fp)) if err != nil { - log.Fatal("Failed to open file", "error", err) + return fmt.Errorf("failed to stat file %q: %w", fp, err) } - defer file.Close() - fileInfo, err := file.Stat() - if err != nil { - log.Fatal("Failed to get file info", "error", err) + if fileInfo.IsDir() { + return fmt.Errorf("%q is a directory", fp) } - fileName := fileInfo.Name() - fileSize := fileInfo.Size() - - uploadPath := path.Join(dirPath, fileName) - ctx = context.WithValue(ctx, ctxkey.ContentLength, fileSize) + uploadPath := path.Join(dirPath, fileInfo.Name()) ctx = tgutil.ExtWithContext(ctx, bot.ExtContext()) - var reader io.Reader var progressUI *UploadProgress - log.Info("Uploading file...", "file", fp, "to", storname, "as", uploadPath) - - if !noProgress && fileSize > 0 { - progressUI = NewUploadProgress(fileName, fileSize) + if !noProgress && fileInfo.Size() > 0 { + progressUI = NewUploadProgress(fileInfo.Name(), fileInfo.Size()) progressUI.Start() - - reader = ioutil.NewProgressReader(file, fileSize, func(read int64, total int64) { - if total > 0 { - progressUI.UpdateProgress(float64(read) / float64(total)) - } - }) - } else { - reader = file } - - if err := stor.Save(ctx, reader, uploadPath); err != nil { + onProgress := func(uploaded, total int64) { + if progressUI != nil && total > 0 { + progressUI.UpdateProgress(float64(uploaded) / float64(total)) + } + } + logger.Info("Uploading file...", "file", fp, "to", storname, "as", uploadPath) + err = upload.SaveFile(ctx, filepath.Clean(fp), uploadPath, upload.Options{ + Storage: stor, + Retries: max(config.C().Retry, 1), + RetryDelay: 3 * time.Second, + OnProgress: onProgress, + }) + if err != nil { if progressUI != nil { progressUI.SetError(err) } - log.Fatal("Failed to upload file", "error", err) + return err } - if progressUI != nil { progressUI.Done() } - log.Info("File uploaded successfully") + logger.Info("File uploaded successfully") return nil } diff --git a/cmd/version.go b/cmd/version.go index 46bd8aef..c8357fa3 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "runtime" @@ -16,8 +17,9 @@ var VersionCmd = &cobra.Command{ Use: "version", Aliases: []string{"v"}, Short: "Print the version number of saveany-bot", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { fmt.Printf("saveany-bot version: %s %s/%s\nBuildTime: %s, Commit: %s\n", config.Version, runtime.GOOS, runtime.GOARCH, config.BuildTime, config.GitCommit) + return nil }, } @@ -25,42 +27,37 @@ var upgradeCmd = &cobra.Command{ Use: "upgrade", Aliases: []string{"up"}, Short: "Upgrade saveany-bot to the latest version", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { v, err := semver.Parse(config.Version) if err != nil { - fmt.Println("Currently in development version or version info injection failed:", err) - return + return fmt.Errorf("current build has no release version: %w", err) } if err := selfupdate.Updatable(); err != nil { - fmt.Println("This build cannot update itself:", err) - return + return fmt.Errorf("this build cannot update itself: %w", err) } latest, found, err := ghselfupdate.DetectLatest(config.GitRepo) if err != nil { - fmt.Println("Error occurred while detecting latest version:", err) - return + return fmt.Errorf("detect latest release: %w", err) } if !found { - fmt.Println("No releases found") - return + return errors.New("no releases found") } if latest.Version.Major != v.Major { - fmt.Printf("Major version upgrade detected: %s -> %s. Please manually download the latest version and check the migration guide.\n", v, latest.Version) - return + return fmt.Errorf("major upgrade %s -> %s requires a manual download and migration review", v, latest.Version) } if latest.Version.Equals(v) || latest.Version.LT(v) { fmt.Println("Current binary is the latest version", config.Version) - return + return nil } fmt.Printf("Updating to version %s...\n", latest.Version) latest, err = selfupdate.Update(v) if err != nil { - fmt.Println("Update failed:", err) - return + return fmt.Errorf("update failed: %w", err) } fmt.Println("Successfully updated to version", latest.Version) fmt.Println("Release note:\n", latest.ReleaseNotes) fmt.Println("Restart saveany-bot to use it.") + return nil }, } diff --git a/cmd/watch/cmd.go b/cmd/watch/cmd.go index aee8b88a..88cfa3e3 100644 --- a/cmd/watch/cmd.go +++ b/cmd/watch/cmd.go @@ -5,12 +5,10 @@ import ( "time" "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/boot" "github.com/krau/SaveAny-Bot/client/bot" - "github.com/krau/SaveAny-Bot/common/cache" - "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" - "github.com/krau/SaveAny-Bot/database" stortype "github.com/krau/SaveAny-Bot/pkg/enums/storage" "github.com/krau/SaveAny-Bot/storage" "github.com/spf13/cobra" @@ -84,13 +82,11 @@ func runWatch(cmd *cobra.Command, _ []string) error { ctx := cmd.Context() logger := log.FromContext(ctx) - configFile := config.GetConfigFile(cmd) - if err := config.Init(ctx, configFile); err != nil { + opts := boot.Parse(cmd) + opts.InitWeb = false + if _, err := boot.Init(ctx, opts); err != nil { return fmt.Errorf("failed to load config: %w", err) } - i18n.Init(config.C().Lang) - cache.Init() - database.Init(ctx) stor, err := storage.GetStorageByName(ctx, storName) if err != nil { @@ -99,7 +95,10 @@ func runWatch(cmd *cobra.Command, _ []string) error { // Telegram storage needs the bot client and its ext context injected into ctx. if stor.Type() == stortype.Telegram { - bot.Init(ctx) + if err := bot.Start(ctx); err != nil { + return fmt.Errorf("failed to start bot: %w", err) + } + defer bot.Stop() ctx = tgutil.ExtWithContext(ctx, bot.ExtContext()) } diff --git a/cmd/watch/uploader.go b/cmd/watch/uploader.go index 66c06c10..2cb177f3 100644 --- a/cmd/watch/uploader.go +++ b/cmd/watch/uploader.go @@ -10,7 +10,7 @@ import ( "time" "github.com/charmbracelet/log" - "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" + uploadpkg "github.com/krau/SaveAny-Bot/common/upload" "github.com/krau/SaveAny-Bot/storage" ) @@ -126,7 +126,7 @@ func (u *Uploader) worker(ctx context.Context) { } func (u *Uploader) process(ctx context.Context, job uploadJob) { - if err := u.uploadWithRetry(ctx, job); err != nil { + if err := u.upload(ctx, job); err != nil { if ctx.Err() != nil { u.clearInflight(job.localPath) return @@ -160,40 +160,8 @@ func (u *Uploader) clearInflight(localPath string) { u.mu.Unlock() } -func (u *Uploader) uploadWithRetry(ctx context.Context, job uploadJob) error { - var lastErr error - for attempt := 1; attempt <= u.retry; attempt++ { - if ctx.Err() != nil { - return ctx.Err() - } - err := u.upload(ctx, job) - if err == nil { - return nil - } - if ctx.Err() != nil { - return ctx.Err() - } - lastErr = err - u.logger.Warnf("upload %s failed (attempt %d/%d): %v", job.localPath, attempt, u.retry, err) - if attempt < u.retry { - select { - case <-time.After(u.retryDelay): - case <-ctx.Done(): - return ctx.Err() - } - } - } - return lastErr -} - func (u *Uploader) upload(ctx context.Context, job uploadJob) error { - file, err := os.Open(filepath.Clean(job.localPath)) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - info, err := file.Stat() + info, err := os.Stat(filepath.Clean(job.localPath)) if err != nil { return fmt.Errorf("failed to stat file: %w", err) } @@ -204,16 +172,25 @@ func (u *Uploader) upload(ctx context.Context, job uploadJob) error { // Keep the relative directory structure on the storage side. storagePath := path.Join(u.destDir, filepath.ToSlash(job.relPath)) - uploadCtx := context.WithValue(ctx, ctxkey.ContentLength, info.Size()) - if u.overwrite { - uploadCtx = storage.WithOverwrite(uploadCtx) - } else if u.stor.Exists(uploadCtx, storagePath) { - u.logger.Infof("skip existing file: %s", storagePath) - return nil + uploadCtx := ctx + if !u.overwrite { + exists, checkErr := u.stor.Exists(uploadCtx, storagePath) + if checkErr != nil { + return fmt.Errorf("failed to check existing file: %w", checkErr) + } + if exists { + u.logger.Infof("skip existing file: %s", storagePath) + return nil + } } u.logger.Infof("uploading %s -> %s (%d bytes)", job.localPath, storagePath, info.Size()) - if err := u.stor.Save(uploadCtx, file, storagePath); err != nil { + if err := uploadpkg.SaveFile(uploadCtx, job.localPath, storagePath, uploadpkg.Options{ + Storage: u.stor, + Overwrite: u.overwrite, + Retries: u.retry, + RetryDelay: u.retryDelay, + }); err != nil { return fmt.Errorf("failed to save to storage: %w", err) } u.logger.Infof("uploaded %s", storagePath) diff --git a/common/i18n/i18n.go b/common/i18n/i18n.go index 48c3e25c..55b0c235 100644 --- a/common/i18n/i18n.go +++ b/common/i18n/i18n.go @@ -2,6 +2,7 @@ package i18n import ( "embed" + "sync" "maps" @@ -15,40 +16,51 @@ import ( var localesFS embed.FS var ( + mu sync.RWMutex bundle *i18n.Bundle localizer *i18n.Localizer ) func Init(lang string) { - bundle = i18n.NewBundle(language.SimplifiedChinese) - bundle.RegisterUnmarshalFunc("yaml", yaml.Unmarshal) + newBundle := i18n.NewBundle(language.SimplifiedChinese) + newBundle.RegisterUnmarshalFunc("yaml", yaml.Unmarshal) files, err := localesFS.ReadDir("locale") if err != nil { panic("failed to read locale directory: " + err.Error()) } for _, file := range files { - if _, err := bundle.LoadMessageFileFS(localesFS, "locale/"+file.Name()); err != nil { + if _, err := newBundle.LoadMessageFileFS(localesFS, "locale/"+file.Name()); err != nil { panic("failed to load message file: " + err.Error()) } } if lang == "" { lang = "zh-Hans" } - localizer = i18n.NewLocalizer(bundle, lang) - if localizer == nil { + newLocalizer := i18n.NewLocalizer(newBundle, lang) + if newLocalizer == nil { panic("failed to create localizer, check your config for valid language setting") } + mu.Lock() + bundle = newBundle + localizer = newLocalizer + mu.Unlock() } func T(key i18nk.Key, templateData ...map[string]any) string { - if localizer == nil || bundle == nil { - Init("zh-Hans") + mu.RLock() + loc := localizer + mu.RUnlock() + if loc == nil { + Init("") + mu.RLock() + loc = localizer + mu.RUnlock() } templateDataMap := make(map[string]any) for _, data := range templateData { maps.Copy(templateDataMap, data) } - msg, err := localizer.Localize(&i18n.LocalizeConfig{ + msg, err := loc.Localize(&i18n.LocalizeConfig{ MessageID: string(key), TemplateData: templateDataMap, }) diff --git a/common/i18n/i18nk/keys.go b/common/i18n/i18nk/keys.go index 7ef3714c..506b2720 100644 --- a/common/i18n/i18nk/keys.go +++ b/common/i18n/i18nk/keys.go @@ -11,31 +11,19 @@ const ( BotMsgAria2InfoAria2DownloadAdded Key = "bot.msg.aria2.info_aria2_download_added" BotMsgAria2InfoSelectStorage Key = "bot.msg.aria2.info_select_storage" BotMsgCancelErrorCancelFailed Key = "bot.msg.cancel.error_cancel_failed" - BotMsgCancelInfoCancelRequested Key = "bot.msg.cancel.info_cancel_requested" BotMsgCancelInfoCancellingTask Key = "bot.msg.cancel.info_cancelling_task" - BotMsgCancelUsage Key = "bot.msg.cancel.usage" - BotMsgCmdAria2dl Key = "bot.msg.cmd.aria2dl" - BotMsgCmdCancel Key = "bot.msg.cmd.cancel" BotMsgCmdConfig Key = "bot.msg.cmd.config" - BotMsgCmdDir Key = "bot.msg.cmd.dir" - BotMsgCmdDl Key = "bot.msg.cmd.dl" - BotMsgCmdFnametmpl Key = "bot.msg.cmd.fnametmpl" - BotMsgCmdHelp Key = "bot.msg.cmd.help" - BotMsgCmdImport Key = "bot.msg.cmd.import" - BotMsgCmdLswatch Key = "bot.msg.cmd.lswatch" + BotMsgCmdDirs Key = "bot.msg.cmd.dirs" + BotMsgCmdDownload Key = "bot.msg.cmd.download" BotMsgCmdParser Key = "bot.msg.cmd.parser" - BotMsgCmdRule Key = "bot.msg.cmd.rule" + BotMsgCmdRules Key = "bot.msg.cmd.rules" BotMsgCmdSave Key = "bot.msg.cmd.save" - BotMsgCmdSilent Key = "bot.msg.cmd.silent" BotMsgCmdStart Key = "bot.msg.cmd.start" - BotMsgCmdStorage Key = "bot.msg.cmd.storage" - BotMsgCmdSyncpeers Key = "bot.msg.cmd.syncpeers" - BotMsgCmdTask Key = "bot.msg.cmd.task" + BotMsgCmdTasks Key = "bot.msg.cmd.tasks" + BotMsgCmdTemplate Key = "bot.msg.cmd.template" BotMsgCmdTransfer Key = "bot.msg.cmd.transfer" - BotMsgCmdUnwatch Key = "bot.msg.cmd.unwatch" BotMsgCmdUpdate Key = "bot.msg.cmd.update" BotMsgCmdWatch Key = "bot.msg.cmd.watch" - BotMsgCmdYtdlp Key = "bot.msg.cmd.ytdlp" BotMsgCommonButtonConflictOverwrite Key = "bot.msg.common.button_conflict_overwrite" BotMsgCommonButtonConflictRename Key = "bot.msg.common.button_conflict_rename" BotMsgCommonButtonConflictSkip Key = "bot.msg.common.button_conflict_skip" @@ -84,23 +72,30 @@ const ( BotMsgCommonPromptSelectDefaultDir Key = "bot.msg.common.prompt_select_default_dir" BotMsgCommonPromptSelectDefaultStorage Key = "bot.msg.common.prompt_select_default_storage" BotMsgCommonPromptSelectDir Key = "bot.msg.common.prompt_select_dir" + BotMsgConfigButtonBack Key = "bot.msg.config.button_back" BotMsgConfigButtonConflictStrategy Key = "bot.msg.config.button_conflict_strategy" + BotMsgConfigButtonDefaultStorage Key = "bot.msg.config.button_default_storage" BotMsgConfigButtonFilenameStrategy Key = "bot.msg.config.button_filename_strategy" + BotMsgConfigButtonSilent Key = "bot.msg.config.button_silent" + BotMsgConfigButtonTemplate Key = "bot.msg.config.button_template" BotMsgConfigConflictStrategyAsk Key = "bot.msg.config.conflict_strategy_ask" BotMsgConfigConflictStrategyOverwrite Key = "bot.msg.config.conflict_strategy_overwrite" BotMsgConfigConflictStrategyRename Key = "bot.msg.config.conflict_strategy_rename" BotMsgConfigConflictStrategySkip Key = "bot.msg.config.conflict_strategy_skip" BotMsgConfigErrorInvalidCallbackData Key = "bot.msg.config.error_invalid_callback_data" BotMsgConfigErrorInvalidTemplate Key = "bot.msg.config.error_invalid_template" - BotMsgConfigFnametmplHelp Key = "bot.msg.config.fnametmpl_help" BotMsgConfigInfoConflictStrategySet Key = "bot.msg.config.info_conflict_strategy_set" BotMsgConfigInfoCurrentTemplatePrefix Key = "bot.msg.config.info_current_template_prefix" BotMsgConfigInfoFilenameStrategySet Key = "bot.msg.config.info_filename_strategy_set" BotMsgConfigInfoTemplateUpdated Key = "bot.msg.config.info_template_updated" + BotMsgConfigMenuHint Key = "bot.msg.config.menu_hint" + BotMsgConfigMenuTitle Key = "bot.msg.config.menu_title" BotMsgConfigPromptSelectConflictStrategy Key = "bot.msg.config.prompt_select_conflict_strategy" BotMsgConfigPromptSelectFilenameStrategy Key = "bot.msg.config.prompt_select_filename_strategy" BotMsgConfigPromptSelectOption Key = "bot.msg.config.prompt_select_option" + BotMsgConfigTemplateHelp Key = "bot.msg.config.template_help" BotMsgDirButtonDefault Key = "bot.msg.dir.button_default" + BotMsgDirButtonDelete Key = "bot.msg.dir.button_delete" BotMsgDirErrorCreateDirFailed Key = "bot.msg.dir.error_create_dir_failed" BotMsgDirErrorDeleteDirFailed Key = "bot.msg.dir.error_delete_dir_failed" BotMsgDirErrorGetUserDirsFailed Key = "bot.msg.dir.error_get_user_dirs_failed" @@ -118,9 +113,17 @@ const ( BotMsgDirHelpUsage Key = "bot.msg.dir.help_usage" BotMsgDirInfoCreateDirSuccess Key = "bot.msg.dir.info_create_dir_success" BotMsgDirInfoDeleteDirSuccess Key = "bot.msg.dir.info_delete_dir_success" - BotMsgDlErrorNoValidLinks Key = "bot.msg.dl.error_no_valid_links" - BotMsgDlInfoFilesSelectStorage Key = "bot.msg.dl.info_files_select_storage" - BotMsgDlUsage Key = "bot.msg.dl.usage" + BotMsgDirMenuEmpty Key = "bot.msg.dir.menu_empty" + BotMsgDirMenuHint Key = "bot.msg.dir.menu_hint" + BotMsgDirMenuTitle Key = "bot.msg.dir.menu_title" + BotMsgDownloadButtonAria2 Key = "bot.msg.download.button_aria2" + BotMsgDownloadButtonDirect Key = "bot.msg.download.button_direct" + BotMsgDownloadButtonVideo Key = "bot.msg.download.button_video" + BotMsgDownloadErrorNoEngine Key = "bot.msg.download.error_no_engine" + BotMsgDownloadErrorNoValidLinks Key = "bot.msg.download.error_no_valid_links" + BotMsgDownloadInfoSelectStorage Key = "bot.msg.download.info_select_storage" + BotMsgDownloadPromptSelectEngine Key = "bot.msg.download.prompt_select_engine" + BotMsgDownloadUsage Key = "bot.msg.download.usage" BotMsgHelpTextFmt Key = "bot.msg.help_text_fmt" BotMsgMediaGroupErrorBuildStorageSelectKeyboardFailed Key = "bot.msg.media_group.error_build_storage_select_keyboard_failed" BotMsgMediaGroupInfoGroupFoundFilesSelectStorage Key = "bot.msg.media_group.info_group_found_files_select_storage" @@ -152,6 +155,7 @@ const ( BotMsgProgressAvgSpeedPrefix Key = "bot.msg.progress.avg_speed_prefix" BotMsgProgressBatchCanceled Key = "bot.msg.progress.batch_canceled" BotMsgProgressBatchDone Key = "bot.msg.progress.batch_done" + BotMsgProgressBatchDoneWithFailed Key = "bot.msg.progress.batch_done_with_failed" BotMsgProgressBatchDoneWithSkipped Key = "bot.msg.progress.batch_done_with_skipped" BotMsgProgressBatchFailedGroup Key = "bot.msg.progress.batch_failed_group" BotMsgProgressBatchFailedItem Key = "bot.msg.progress.batch_failed_item" @@ -219,6 +223,11 @@ const ( BotMsgProgressYtdlpDone Key = "bot.msg.progress.ytdlp_done" BotMsgProgressYtdlpDownloading Key = "bot.msg.progress.ytdlp_downloading" BotMsgProgressYtdlpStart Key = "bot.msg.progress.ytdlp_start" + BotMsgRetryErrorFailed Key = "bot.msg.retry.error_failed" + BotMsgRetryInfoStarted Key = "bot.msg.retry.info_started" + BotMsgRuleButtonDelete Key = "bot.msg.rule.button_delete" + BotMsgRuleButtonDisable Key = "bot.msg.rule.button_disable" + BotMsgRuleButtonEnable Key = "bot.msg.rule.button_enable" BotMsgRuleErrorCreateRuleFailed Key = "bot.msg.rule.error_create_rule_failed" BotMsgRuleErrorDeleteRuleFailed Key = "bot.msg.rule.error_delete_rule_failed" BotMsgRuleErrorGetUserRulesFailed Key = "bot.msg.rule.error_get_user_rules_failed" @@ -240,6 +249,9 @@ const ( BotMsgRuleInfoPresetImported Key = "bot.msg.rule.info_preset_imported" BotMsgRuleInfoRuleModeDisabled Key = "bot.msg.rule.info_rule_mode_disabled" BotMsgRuleInfoRuleModeEnabled Key = "bot.msg.rule.info_rule_mode_enabled" + BotMsgRuleMenuEmpty Key = "bot.msg.rule.menu_empty" + BotMsgRuleMenuHint Key = "bot.msg.rule.menu_hint" + BotMsgRuleMenuTitle Key = "bot.msg.rule.menu_title" BotMsgRulePromptProvideRuleId Key = "bot.msg.rule.prompt_provide_rule_id" BotMsgRulePromptProvideStorageName Key = "bot.msg.rule.prompt_provide_storage_name" BotMsgSaveErrorInvalidIdOrUsername Key = "bot.msg.save.error_invalid_id_or_username" @@ -249,8 +261,11 @@ const ( BotMsgSyncpeersFailed Key = "bot.msg.syncpeers.failed" BotMsgSyncpeersStart Key = "bot.msg.syncpeers.start" BotMsgSyncpeersSuccess Key = "bot.msg.syncpeers.success" + BotMsgTasksCancelButton Key = "bot.msg.tasks.cancel_button" BotMsgTasksCancelFailed Key = "bot.msg.tasks.cancel_failed" BotMsgTasksCancelRequestedPrefix Key = "bot.msg.tasks.cancel_requested_prefix" + BotMsgTasksFailedEmpty Key = "bot.msg.tasks.failed_empty" + BotMsgTasksFailedTitle Key = "bot.msg.tasks.failed_title" BotMsgTasksFieldCreated Key = "bot.msg.tasks.field_created" BotMsgTasksFieldId Key = "bot.msg.tasks.field_id" BotMsgTasksFieldStatus Key = "bot.msg.tasks.field_status" @@ -259,8 +274,15 @@ const ( BotMsgTasksInfoAddedToQueuePrefix Key = "bot.msg.tasks.info_added_to_queue_prefix" BotMsgTasksInfoFilenamePrefix Key = "bot.msg.tasks.info_filename_prefix" BotMsgTasksInfoQueueLengthPrefix Key = "bot.msg.tasks.info_queue_length_prefix" + BotMsgTasksMenuBack Key = "bot.msg.tasks.menu_back" + BotMsgTasksMenuFailed Key = "bot.msg.tasks.menu_failed" + BotMsgTasksMenuHint Key = "bot.msg.tasks.menu_hint" + BotMsgTasksMenuQueued Key = "bot.msg.tasks.menu_queued" + BotMsgTasksMenuRunning Key = "bot.msg.tasks.menu_running" + BotMsgTasksMenuTitle Key = "bot.msg.tasks.menu_title" BotMsgTasksQueuedEmpty Key = "bot.msg.tasks.queued_empty" BotMsgTasksQueuedTitle Key = "bot.msg.tasks.queued_title" + BotMsgTasksRetryButton Key = "bot.msg.tasks.retry_button" BotMsgTasksRunningEmpty Key = "bot.msg.tasks.running_empty" BotMsgTasksRunningTitle Key = "bot.msg.tasks.running_title" BotMsgTasksStatusCancelRequested Key = "bot.msg.tasks.status_cancel_requested" @@ -268,8 +290,6 @@ const ( BotMsgTasksStatusRunning Key = "bot.msg.tasks.status_running" BotMsgTasksTotalPrefix Key = "bot.msg.tasks.total_prefix" BotMsgTasksTruncatedNote Key = "bot.msg.tasks.truncated_note" - BotMsgTasksUsage Key = "bot.msg.tasks.usage" - BotMsgTasksUsageCancel Key = "bot.msg.tasks.usage_cancel" BotMsgTelegraphErrorBuildStorageSelectKeyboardFailed Key = "bot.msg.telegraph.error_build_storage_select_keyboard_failed" BotMsgTelegraphInfoPicCountPrefix Key = "bot.msg.telegraph.info_pic_count_prefix" BotMsgTelegraphInfoPromptSelectStorage Key = "bot.msg.telegraph.info_prompt_select_storage" @@ -303,9 +323,7 @@ const ( BotMsgUpdateInfoUpgradingWithVersion Key = "bot.msg.update.info_upgrading_with_version" BotMsgWatchErrorFilterFormatInvalid Key = "bot.msg.watch.error_filter_format_invalid" BotMsgWatchErrorFilterTypeUnsupported Key = "bot.msg.watch.error_filter_type_unsupported" - BotMsgWatchErrorNotifyArgInvalid Key = "bot.msg.watch.error_notify_arg_invalid" BotMsgWatchErrorUnwatchChatFailed Key = "bot.msg.watch.error_unwatch_chat_failed" - BotMsgWatchErrorUnwatchNoChatProvided Key = "bot.msg.watch.error_unwatch_no_chat_provided" BotMsgWatchErrorWatchChatFailed Key = "bot.msg.watch.error_watch_chat_failed" BotMsgWatchInfoAlreadyWatchingChat Key = "bot.msg.watch.info_already_watching_chat" BotMsgWatchInfoNotifyDisabled Key = "bot.msg.watch.info_notify_disabled" @@ -315,13 +333,16 @@ const ( BotMsgWatchInfoWatchListEmpty Key = "bot.msg.watch.info_watch_list_empty" BotMsgWatchInfoWatchListFilterPrefix Key = "bot.msg.watch.info_watch_list_filter_prefix" BotMsgWatchInfoWatchListHeader Key = "bot.msg.watch.info_watch_list_header" + BotMsgWatchMenuAdd Key = "bot.msg.watch.menu_add" + BotMsgWatchMenuAddHelp Key = "bot.msg.watch.menu_add_help" + BotMsgWatchMenuBack Key = "bot.msg.watch.menu_back" + BotMsgWatchMenuHint Key = "bot.msg.watch.menu_hint" + BotMsgWatchMenuList Key = "bot.msg.watch.menu_list" + BotMsgWatchMenuNotify Key = "bot.msg.watch.menu_notify" + BotMsgWatchMenuStop Key = "bot.msg.watch.menu_stop" + BotMsgWatchMenuSync Key = "bot.msg.watch.menu_sync" + BotMsgWatchMenuTitle Key = "bot.msg.watch.menu_title" BotMsgWatchNotifyDetected Key = "bot.msg.watch.notify_detected" - BotMsgWatchHelpText Key = "bot.msg.watch_help_text" - BotMsgYtdlpErrorDownloadFailed Key = "bot.msg.ytdlp.error_download_failed" - BotMsgYtdlpErrorNoValidUrls Key = "bot.msg.ytdlp.error_no_valid_urls" - BotMsgYtdlpInfoDownloading Key = "bot.msg.ytdlp.info_downloading" - BotMsgYtdlpInfoUrlsSelectStorage Key = "bot.msg.ytdlp.info_urls_select_storage" - BotMsgYtdlpUsage Key = "bot.msg.ytdlp.usage" ConfigErrDuplicateStorageName Key = "config.err.duplicate_storage_name" ConfigErrInvalidCacheDir Key = "config.err.invalid_cache_dir" ErrCleanCacheFailed Key = "err.clean_cache_failed" diff --git a/common/i18n/locale/en.yaml b/common/i18n/locale/en.yaml index dd50608a..25a15d4b 100644 --- a/common/i18n/locale/en.yaml +++ b/common/i18n/locale/en.yaml @@ -24,48 +24,33 @@ bot: Version: %s , Commit: %s Commands: - /start - Start using the bot - /help - Show help - /silent - Toggle silent mode - /storage - Set default storage - /save [custom filename] - Save file - /import [channel_id] [filter] - Import files from storage to Telegram - /dir - Manage storage directories - /rule - Manage rules - /config - Modify configuration - /fnametmpl - Set custom filename template + /start - Open help and command list + /save - Save replied media or batch messages + /download - Download links (direct, Aria2, or video) + /transfer - Transfer files between storages + /tasks - Manage tasks + /config - User settings (storage, silent mode, naming) + /template - Set filename template + /dirs - Manage storage directories + /rules - Manage storage rules + /watch - Watch chats and notifications (UserBot) /parser - Manage parser plugins - /task - Manage task queue - /watch - Watch chats and auto save (UserBot) - /unwatch - Stop watching chats (UserBot) - /lswatch - List watched chats (UserBot) - /syncpeers - Sync peer chats (UserBot) - /update - Check and upgrade to latest version + /update - Check for updates Usage guide: https://sabot.unv.app/usage cmd: - start: "Start using" - silent: "Toggle silent mode" - storage: "Set default storage" - dir: "Manage storage directories" - rule: "Manage auto-save rules" + start: "Open help" save: "Save files" - dl: "Download files from given links" - aria2dl: "Download files using Aria2" - ytdlp: "Download video/audio using yt-dlp" - import: "Import files from storage to Telegram" + download: "Download links (direct, Aria2, video)" transfer: "Transfer files between storages" - task: "Manage task queue" - cancel: "Cancel task" - watch: "Watch chats (UserBot)" - unwatch: "Stop watching chats (UserBot)" - lswatch: "List watched chats (UserBot)" - config: "Modify configuration" - fnametmpl: "Set filename template" - help: "Show help" + tasks: "Manage tasks" + config: "User settings" + template: "Set filename template" + dirs: "Manage storage directories" + rules: "Manage auto-save rules" + watch: "Watch chats and notifications (UserBot)" parser: "Manage parsers" update: "Check for updates" - syncpeers: "Sync peer chats (UserBot)" save_help_text: | Usage: @@ -76,22 +61,6 @@ bot: 2. After setting default storage, send /save to batch save files. Rules will be applied; if no rule matches, default storage will be used. Example: /save @acherkrau 114-514 - watch_help_text: | - Use /watch to watch messages in a chat and automatically save them to the default storage, following storage rules. - - Syntax: - /watch [filter] - - Parameters: - - : Chat ID or username - - [filter]: Optional, format is filter_type:expression , see docs for all supported filters - - Example: - /watch -1002229835658 msgre:.*plana.* - - This will watch chat with ID -1002229835658 and save all media messages containing "plana". - - Notifications for saved messages are off by default, toggle them with /watch notify on|off. common: cancel_button_text: "Cancel" error_invalid_regex: "Invalid regex: {{.Error}}" @@ -106,7 +75,7 @@ bot: info_found_files_select_storage: "Found {{.Count}} files, please select storage" error_get_user_failed: "Failed to get user" error_get_user_with_err_failed: "Failed to get user: {{.Error}}" - error_default_storage_not_set: "Please set a default storage first with /storage" + error_default_storage_not_set: "Please set a default storage first in /config" error_no_available_storage: "No available storage" error_get_storage_failed: "Failed to get storage: {{.Error}}" prompt_select_default_storage: "Please select a storage to set as default" @@ -146,6 +115,18 @@ bot: save: error_invalid_id_or_username: "Invalid ID or username: {{.Error}}" watch: + menu_title: "Watch manager" + menu_hint: "Manage watched chats, notifications, and peer sync." + menu_add: "Add watch" + menu_list: "Watched chats" + menu_notify: "Notifications" + menu_sync: "Sync peers" + menu_stop: "Stop" + menu_back: "Back" + menu_add_help: | + Send /watch [filter] to start watching. + Example: + /watch -1002229835658 msgre:.*plana.* error_filter_format_invalid: "Invalid filter format, please use :" error_filter_type_unsupported: "Unsupported filter type, please see the docs" error_watch_chat_failed: "Failed to watch chat: {{.Error}}" @@ -154,16 +135,22 @@ bot: info_watch_list_empty: "No chats are being watched currently" info_watch_list_header: "Currently watched chats:\n" info_watch_list_filter_prefix: " (filter: " - error_unwatch_no_chat_provided: "Please provide a chat ID or username to unwatch" error_unwatch_chat_failed: "Failed to unwatch chat: {{.Error}}" info_watch_chat_stopped: "Stopped watching chat: {{.Chat}}" - error_notify_arg_invalid: "Invalid argument, usage: /watch notify on|off" - info_notify_enabled: "Task notifications are enabled for every chat you watch: each saved message and its result is reported here.\n\nUsage: /watch notify on|off" - info_notify_disabled: "Task notifications are disabled for every chat you watch.\n\nUsage: /watch notify on|off" + info_notify_enabled: "Task notifications are enabled for every chat you watch: each saved message and its result is reported here." + info_notify_disabled: "Task notifications are disabled for every chat you watch." notify_detected: "🔔 Watched message detected\n{{.Name}}\nProgress will be reported here." tasks: - usage_cancel: "Usage: /tasks cancel " - usage: "Usage: /tasks [running|queued|cancel ]" + menu_title: "Task manager" + menu_hint: "Inspect running, queued, and failed tasks." + menu_running: "Running" + menu_queued: "Queued" + menu_failed: "Failed" + menu_back: "Back" + failed_title: "Failed tasks" + failed_empty: "No failed tasks" + retry_button: "Retry" + cancel_button: "Cancel" cancel_failed: "Failed to cancel task: {{.Error}}" cancel_requested_prefix: "Cancel requested for task: " running_empty: "No running tasks" @@ -184,6 +171,12 @@ bot: info_filename_prefix: "Filename: " info_queue_length_prefix: "\nCurrent queued tasks: " rule: + menu_title: "Storage rules" + menu_hint: "Use /rules add to add a rule." + menu_empty: "No rules configured" + button_enable: "Enable rules" + button_disable: "Disable rules" + button_delete: "Delete" error_get_user_rules_failed: "Failed to get user rules" error_update_user_failed: "Failed to update user" info_rule_mode_enabled: "Rule mode enabled" @@ -195,19 +188,23 @@ bot: error_invalid_rule_id: "Invalid rule ID" error_delete_rule_failed: "Failed to delete rule" info_delete_rule_success: "Rule deleted successfully" - help_usage: "Usage: /rule " + help_usage: "Usage: /rules " help_current_mode_enabled: "\nRule mode is currently enabled" help_current_mode_disabled: "\nRule mode is currently disabled" help_available_ops: "\n\nAvailable operations:\n" help_switch_suffix: " - Toggle rule mode\n" - help_add_suffix: " - Add rule\n" + help_add_suffix: " - Add rule\n" help_del_suffix: " - Delete rule\n" - help_preset_suffix: " [base_path] - Import built-in filetype rules (video/image/audio/document/archive)\n" + help_preset_suffix: " [base_path] - Import built-in filetype rules (video/image/audio/document/archive)\n" help_existing_rules_prefix: "\nCurrent rules:\n" prompt_provide_storage_name: "Please provide a storage name" error_storage_not_found: "Storage not found: {{.Storage}}" info_preset_imported: "Imported {{.Count}} built-in classification rules into storage {{.Storage}}" dir: + menu_title: "Storage directories" + menu_hint: "Use /dirs add to add a directory." + menu_empty: "No directories configured" + button_delete: "Delete" error_get_user_dirs_failed: "Failed to get user directories" error_get_user_failed: "Failed to get user" error_create_dir_failed: "Failed to create directory" @@ -216,14 +213,14 @@ bot: error_delete_dir_failed: "Failed to delete directory" info_delete_dir_success: "Directory deleted successfully" error_unknown_operation: "Unknown operation" - help_usage: "Usage: /dir " + help_usage: "Usage: /dirs " help_available_ops: "\n\nAvailable operations:\n" help_add_suffix: " - Add path\n" help_del_suffix: " - Delete path\n" help_add_example_prefix: "\nAdd path example:\n" - help_add_example_cmd: "/dir add local1 path/to/dir" + help_add_example_cmd: "/dirs add local1 path/to/dir" help_del_example_prefix: "\n\nDelete path example:\n" - help_del_example_cmd: "/dir del 3" + help_del_example_cmd: "/dirs del 3" help_existing_dirs_prefix: "\n\nCurrent paths:\n" button_default: "Default" parser: @@ -282,7 +279,13 @@ bot: info_upgrade_success: "Upgraded to version {{.Version}}\nIf the bot did not restart automatically, please start it manually" button_upgrade: "Upgrade" config: + menu_title: "User configuration" + menu_hint: "Choose a setting to change." prompt_select_option: "Please select an option to configure" + button_silent: "Silent mode" + button_default_storage: "Default storage" + button_template: "Filename template" + button_back: "Back" button_filename_strategy: "Filename strategy" button_conflict_strategy: "Duplicate file strategy" error_invalid_callback_data: "Invalid callback data" @@ -295,9 +298,9 @@ bot: conflict_strategy_ask: "Ask every time" conflict_strategy_overwrite: "Always overwrite" conflict_strategy_skip: "Always skip" - fnametmpl_help: |- - Use this command to set filename template, for example: - /fnametmpl Image_{{"{{.msgid}}"}}_{{"{{.msgdate}}"}}.jpg + template_help: |- + Use /template to set a filename template, for example: + /template Image_{{"{{.msgid}}"}}_{{"{{.msgdate}}"}}.jpg Available variables: - {{"{{.msgid}}"}}: Message ID @@ -312,16 +315,21 @@ bot: If template parsing fails, it will fall back to default filename. info_template_updated: "Filename template updated" info_current_template_prefix: "Current template: {{.Template}}" - dl: - usage: "Usage: /dl ..." + download: + usage: | + Usage: /download [yt-dlp options] + Choose direct download, Aria2, or video extraction. + Examples: + /download https://example.com/file.zip + /download https://www.youtube.com/watch?v=... + /download --format best https://www.youtube.com/watch?v=... + prompt_select_engine: "Found {{.Count}} link(s), choose a download engine" + button_direct: "Direct" + button_aria2: "Aria2" + button_video: "Video (yt-dlp)" + info_select_storage: "Choose storage for {{.Count}} link(s)" error_no_valid_links: "No valid links to download" - info_files_select_storage: "Total {{.Count}} files, please select storage" - ytdlp: - usage: "Usage: /ytdlp [OPTIONS] [URL2] ...\nExamples:\n /ytdlp https://example.com/video\n /ytdlp --format best https://example.com/video\n /ytdlp --extract-audio --audio-format mp3 https://example.com/video" - error_no_valid_urls: "No valid URLs" - info_urls_select_storage: "Found {{.Count}} links, please select storage" - info_downloading: "Downloading via yt-dlp..." - error_download_failed: "yt-dlp download failed: {{.Error}}" + error_no_engine: "No available download engine for these links" transfer: usage: | Usage: /transfer :/ [filter] @@ -344,10 +352,11 @@ bot: start_stats: "Total files: {{.Count}}\nTotal size: {{.SizeMB}} MB" info_files_select_storage: "Total {{.Count}} files ({{.SizeMB}} MB), please select target storage" error_build_storage_select_keyboard_failed: "Failed to build storage selection keyboard: {{.Error}}" + retry: + info_started: "Retry task created: {{.TaskID}} ({{.Count}} files)" + error_failed: "Failed to retry task: {{.Error}}" cancel: - usage: "Usage: /cancel " error_cancel_failed: "Failed to cancel task: {{.Error}}" - info_cancel_requested: "Cancel requested for task: {{.TaskID}}" info_cancelling_task: "Cancelling task..." media_group: info_saving_files: "Saving files..." @@ -371,6 +380,7 @@ bot: batch_summary_skipped: "⏭️ Skipped: {{.Count}}" batch_done: "✅ Completed\n\nFiles: {{.Count}}\nTotal size: {{.Size}}" batch_done_with_skipped: "⚠️ Completed\n\nSucceeded: {{.Success}}\nSkipped: {{.Skipped}}\nTotal size: {{.Size}}" + batch_done_with_failed: "⚠️ Completed with failures\n\nSucceeded: {{.Success}}\nFailed: {{.Failed}}\nSkipped: {{.Skipped}}\nTotal size: {{.Size}}{{.List}}" batch_canceled: "🚫 Task canceled\n\nFiles: {{.Total}}\nCompleted: {{.Completed}}\nIncomplete: {{.Incomplete}}\nSkipped: {{.Skipped}}" batch_failed_item: "❌ Processing failed\n\nFailed file: {{.Index}}. {{.Name}}\nStage: {{.Stage}}\nProgress: {{.Progress}}\nSpeed before failure: {{.Speed}}\nReason: {{.Reason}}\n\n✅ Completed: {{.Completed}}\n❌ Failed: {{.Failed}}\n⏹️ Incomplete: {{.Incomplete}}" batch_failed_group: "❌ Batch upload failed\n\nAffected files: {{.Affected}}\nReason: {{.Reason}}\n\n✅ Completed: {{.Completed}}\n❌ Batch failed: {{.Failed}}\n⏹️ Incomplete: {{.Incomplete}}" diff --git a/common/i18n/locale/zh-Hans.yaml b/common/i18n/locale/zh-Hans.yaml index b8057dce..ff1d0cd8 100644 --- a/common/i18n/locale/zh-Hans.yaml +++ b/common/i18n/locale/zh-Hans.yaml @@ -24,75 +24,43 @@ bot: 版本: %s , 提交: %s 命令: - /start - 开始使用 - /help - 显示帮助 - /silent - 开关静默模式 - /storage - 设置默认存储位置 - /save [自定义文件名] - 保存文件 - /dl <链接1> <链接2> ... - 下载给定链接的文件 - /import <存储名> <目录路径> [频道ID] [过滤器] - 从存储端导入文件到 Telegram - /dir - 管理存储目录 - /rule - 管理规则 - /config - 修改配置 - /fnametmpl - 设置文件自定义命名模板 + /start - 打开帮助和命令列表 + /save - 保存回复的媒体或批量消息 + /download - 下载链接(直链、Aria2 或视频) + /transfer - 在存储端之间传输文件 + /tasks - 管理任务 + /config - 用户设置(存储、静默模式、命名) + /template - 设置文件名模板 + /dirs - 管理存储目录 + /rules - 管理存储规则 + /watch - 监听聊天和通知(UserBot) /parser - 管理解析器插件 - /task - 管理任务队列 - /watch - 监听聊天并自动保存 (UserBot) - /unwatch - 取消监听聊天 (UserBot) - /lswatch - 列出正在监听的聊天 (UserBot) - /syncpeers - 同步对话列表 (UserBot) - /update - 检查更新并升级 + /update - 检查更新 使用帮助: https://sabot.unv.app/usage cmd: - start: "开始使用" - silent: "切换静默模式" - storage: "设置默认存储端" - dir: "管理存储文件夹" - rule: "管理自动存储规则" + start: "打开帮助" save: "保存文件" - dl: "下载给定链接的文件" - aria2dl: "使用 Aria2 下载给定链接的文件" - ytdlp: "使用 yt-dlp 下载视频/音频" - import: "从存储端导入文件到 Telegram" + download: "下载链接(直链、Aria2、视频)" transfer: "在存储端之间传输文件" - task: "管理任务队列" - cancel: "取消任务" - watch: "监听聊天(UserBot)" - unwatch: "取消监听聊天(UserBot)" - lswatch: "列出监听的聊天(UserBot)" - syncpeers: "同步对话列表(UserBot)" - config: "修改配置" - fnametmpl: "设置文件命名模板" - help: "显示帮助" + tasks: "管理任务" + config: "用户设置" + template: "设置文件名模板" + dirs: "管理存储目录" + rules: "管理自动存储规则" + watch: "监听聊天和通知(UserBot)" parser: "管理解析器" update: "检查更新" save_help_text: | 使用方法: - 1. 使用该命令回复要保存的文件, 可选文件名参数. + 1. 回复要保存的文件消息,可选指定文件名。 示例: /save custom_file_name.mp4 - 2. 设置默认存储后, 发送 /save <频道ID/用户名> <消息ID范围> 来批量保存文件. 遵从存储规则, 若未匹配到任何规则则使用默认存储. + 2. 设置默认存储后,发送 /save <频道ID/用户名> <消息ID范围> 批量保存文件。优先匹配存储规则,未命中时使用默认存储。 示例: /save @acherkrau 114-514 - watch_help_text: | - 使用 /watch 命令监听一个聊天的消息, 并自动保存到默认存储中, 遵从存储规则. - - 命令语法: - /watch [filter] - - 参数: - - : 聊天的 ID 或用户名 - - [filter]: 可选, 格式为 过滤器类型:表达式 , 所有支持类型的过滤器请查看文档 - - 命令示例: - /watch -1002229835658 msgre:.*plana.* - - 这将监听 ID 为 -1002229835658 的聊天, 并转存所有包含 "plana" 的媒体消息 - - 监听任务的通知默认关闭, 可使用 /watch notify on|off 开启或关闭. common: cancel_button_text: "取消任务" error_invalid_regex: "无效的正则表达式: {{.Error}}" @@ -100,14 +68,14 @@ bot: error_invalid_id_or_username: "无效的ID或用户名: {{.Error}}" error_get_messages_failed: "获取消息失败: {{.Error}}" info_fetching_messages: "正在获取消息..." - error_no_messages_in_range: "没有找到指定范围内的消息" - error_no_savable_messages_in_range: "没有找到指定范围内的可保存消息" + error_no_messages_in_range: "未找到指定范围内的消息" + error_no_savable_messages_in_range: "指定范围内未找到可保存的消息" error_build_storage_select_message_failed: "构建存储选择消息失败: {{.Error}}" error_build_storage_select_keyboard_failed: "构建存储选择键盘失败: {{.Error}}" info_found_files_select_storage: "找到 {{.Count}} 个文件, 请选择存储位置" error_get_user_failed: "获取用户失败" error_get_user_with_err_failed: "获取用户失败: {{.Error}}" - error_default_storage_not_set: "请先设置默认存储, 使用 /storage 命令" + error_default_storage_not_set: "请先在 /config 中设置默认存储" error_no_available_storage: "无可用的存储" error_get_storage_failed: "获取存储失败: {{.Error}}" prompt_select_default_storage: "请选择要设为默认的存储位置" @@ -135,39 +103,57 @@ bot: info_silent_mode_off: "已关闭静默模式" error_get_file_failed: "获取文件失败: {{.Error}}" info_fetching_file_info: "正在获取文件信息..." - error_no_savable_files_found: "没有找到可保存的文件" + error_no_savable_files_found: "未找到可保存的文件" error_parse_telegraph_path_failed: "解析 telegraph 路径失败: {{.Error}}" info_fetching_telegraph_page: "正在获取 telegraph 页面..." error_get_telegraph_page_failed: "获取 telegraph 页面失败: {{.Error}}" - error_no_images_in_telegraph_page: "在 telegraph 页面中未找到图片" + error_no_images_in_telegraph_page: "Telegraph 页面中未找到图片" error_build_dir_select_keyboard_failed: "构建目录选择键盘失败: {{.Error}}" error_no_permission: | - 您不在白名单中, 无法使用此 Bot. - 您可以部署自己的实例: https://github.com/krau/SaveAny-Bot + 当前账号未在白名单中,无法使用此 Bot。 + 可自行部署实例: https://github.com/krau/SaveAny-Bot save: error_invalid_id_or_username: "无效的ID或用户名: {{.Error}}" watch: + menu_title: "监听管理" + menu_hint: "管理监听聊天、通知和对话同步。" + menu_add: "添加监听" + menu_list: "监听列表" + menu_notify: "通知开关" + menu_sync: "同步对话" + menu_stop: "停止" + menu_back: "返回" + menu_add_help: | + 发送 /watch <聊天ID/用户名> [过滤器] 开始监听。 + 示例: + /watch -1002229835658 msgre:.*plana.* error_filter_format_invalid: "过滤器格式错误, 请使用 <过滤器类型>:<表达式>" error_filter_type_unsupported: "不支持的过滤器类型, 请参阅文档" error_watch_chat_failed: "监听聊天失败: {{.Error}}" info_watch_chat_started: "已开始监听聊天: {{.Chat}}" info_already_watching_chat: "已经在监听此聊天" - info_watch_list_empty: "当前没有监听任何聊天" + info_watch_list_empty: "当前未监听任何聊天" info_watch_list_header: "当前监听的聊天:\n" info_watch_list_filter_prefix: " (过滤器: " - error_unwatch_no_chat_provided: "请提供要取消监听的聊天ID或用户名" error_unwatch_chat_failed: "取消监听聊天失败: {{.Error}}" info_watch_chat_stopped: "已取消监听聊天: {{.Chat}}" - error_notify_arg_invalid: "无效的参数, 用法: /watch notify on|off" - info_notify_enabled: "已为你监听的所有聊天开启任务通知: 监听到的消息及其保存结果会在此处反馈.\n\n用法: /watch notify on|off" - info_notify_disabled: "已为你监听的所有聊天关闭任务通知.\n\n用法: /watch notify on|off" - notify_detected: "🔔 监听到新消息\n{{.Name}}\n进度将在此处更新" + info_notify_enabled: "已开启任务通知,监听到的消息及保存结果将在此反馈。" + info_notify_disabled: "已关闭任务通知。" + notify_detected: "🔔 监听到新消息\n{{.Name}}" tasks: - usage_cancel: "用法: /tasks cancel " - usage: "用法: /tasks [running|queued|cancel ]" + menu_title: "任务管理" + menu_hint: "查看运行中、排队中和失败任务。" + menu_running: "运行中" + menu_queued: "排队中" + menu_failed: "失败任务" + menu_back: "返回" + failed_title: "失败任务" + failed_empty: "暂无失败任务" + retry_button: "重试" + cancel_button: "取消" cancel_failed: "取消任务失败: {{.Error}}" cancel_requested_prefix: "已请求取消任务: " - running_empty: "当前没有正在运行的任务" + running_empty: "暂无运行中的任务" running_title: "当前正在运行的任务:" total_prefix: "总数: {{.Count}}\n" field_id: "ID: " @@ -177,7 +163,7 @@ bot: status_running: "运行中" status_queued: "排队中" status_cancel_requested: "已请求取消" - queued_empty: "当前没有排队中的任务" + queued_empty: "暂无排队中的任务" queued_title: "当前排队中的任务:" truncated_note: "...\n只显示前 10 个任务, 共 {{.Count}} 个任务" info_added_to_queue_full: "已添加到任务队列\n文件名: {{.Filename}}\n当前排队任务数: {{.QueueLength}}" @@ -185,6 +171,12 @@ bot: info_filename_prefix: "文件名: " info_queue_length_prefix: "\n当前排队任务数: " rule: + menu_title: "存储规则" + menu_hint: "使用 /rules add <类型> <内容> <存储> <路径> 添加规则。" + menu_empty: "尚未配置规则" + button_enable: "启用规则" + button_disable: "停用规则" + button_delete: "删除" error_get_user_rules_failed: "获取用户规则失败" error_update_user_failed: "更新用户失败" info_rule_mode_enabled: "已启用规则模式" @@ -196,19 +188,23 @@ bot: error_invalid_rule_id: "无效的规则ID" error_delete_rule_failed: "删除规则失败" info_delete_rule_success: "删除规则成功" - help_usage: "使用方法: /rule <操作> <参数...>" + help_usage: "使用方法: /rules <操作> <参数...>" help_current_mode_enabled: "\n当前已启用规则模式" help_current_mode_disabled: "\n当前已禁用规则模式" help_available_ops: "\n\n可用操作:\n" help_switch_suffix: " - 开关规则模式\n" - help_add_suffix: " <类型> <数据> <存储名> <路径> - 添加规则\n" + help_add_suffix: " <类型> <数据> <存储ID|名称> <路径> - 添加规则\n" help_del_suffix: " <规则ID> - 删除规则\n" - help_preset_suffix: " <存储名> [基础路径] - 导入内置文件类型分类规则(视频/图片/音频/文档/压缩包)\n" + help_preset_suffix: " <存储ID|名称> [基础路径] - 导入内置文件类型分类规则(视频/图片/音频/文档/压缩包)\n" help_existing_rules_prefix: "\n当前已添加的规则:\n" prompt_provide_storage_name: "请提供存储名称" error_storage_not_found: "未找到存储: {{.Storage}}" info_preset_imported: "已导入 {{.Count}} 条内置分类规则到存储 {{.Storage}}" dir: + menu_title: "存储目录" + menu_hint: "使用 /dirs add <存储> <路径> 添加目录。" + menu_empty: "尚未配置目录" + button_delete: "删除" error_get_user_dirs_failed: "获取用户文件夹失败" error_get_user_failed: "获取用户失败" error_create_dir_failed: "创建文件夹失败" @@ -217,14 +213,14 @@ bot: error_delete_dir_failed: "删除文件夹失败" info_delete_dir_success: "文件夹删除成功" error_unknown_operation: "未知操作" - help_usage: "使用方法: /dir <操作> <参数...>" + help_usage: "使用方法: /dirs <操作> <参数...>" help_available_ops: "\n\n可用操作:\n" help_add_suffix: " <存储名> <路径> - 添加路径\n" help_del_suffix: " <路径ID> - 删除路径\n" help_add_example_prefix: "\n添加路径示例:\n" - help_add_example_cmd: "/dir add local1 path/to/dir" + help_add_example_cmd: "/dirs add local1 path/to/dir" help_del_example_prefix: "\n\n删除路径示例:\n" - help_del_example_cmd: "/dir del 3" + help_del_example_cmd: "/dirs del 3" help_existing_dirs_prefix: "\n\n当前已添加的路径:\n" button_default: "默认" parser: @@ -261,14 +257,14 @@ bot: update: error_version_var_invalid: "当前处于开发版本或版本信息注入失败: {{.Error}}" error_check_latest_failed: "检测最新版本失败: {{.Error}}" - error_no_release_found: "没有找到版本信息" - info_major_upgrade_required: "检测到大版本更新: {{.Current}} -> {{.Latest}} , 请前往 GitHub 手动下载最新版本并查看迁移指南" - info_already_latest: "当前已经是最新版本: {{.Version}}" + error_no_release_found: "未找到版本信息" + info_major_upgrade_required: "检测到大版本更新: {{.Current}} -> {{.Latest}},请前往 GitHub 查看迁移指南并手动升级" + info_already_latest: "当前已是最新版本: {{.Version}}" info_new_version_in_docker: |- 发现新版本: {{.Latest}} 当前版本: {{.Current}} 发布时间: {{.PublishedAt}} - 由于您正在使用 Docker 部署, 请自行在部署平台上执行更新命令 + 当前为 Docker 部署,请拉取新镜像完成更新 info_new_version_prompt_upgrade: |- 发现新版本: {{.Latest}} 当前版本: {{.Current}} @@ -277,13 +273,19 @@ bot: 下载链接: {{.URL}} 发布时间: {{.PublishedAt}} - 升级将重启 Bot , 是否升级? + 升级将重启 Bot,是否继续? info_upgrading_with_version: "正在升级中, 当前版本: {{.Current}}" error_upgrade_failed: "升级失败: {{.Error}}" - info_upgrade_success: "已升级至版本 {{.Version}}\n若 Bot 未自动重启请手动启动" + info_upgrade_success: "已升级至版本 {{.Version}}\n若未自动重启,请手动启动" button_upgrade: "升级" config: + menu_title: "用户设置" + menu_hint: "选择要修改的设置。" prompt_select_option: "请选择要配置的选项" + button_silent: "静默模式" + button_default_storage: "默认存储" + button_template: "文件名模板" + button_back: "返回" button_filename_strategy: "文件名策略" button_conflict_strategy: "重名文件保存策略" error_invalid_callback_data: "无效的回调数据" @@ -296,9 +298,9 @@ bot: conflict_strategy_ask: "每次询问" conflict_strategy_overwrite: "始终覆盖" conflict_strategy_skip: "始终跳过" - fnametmpl_help: |- + template_help: |- 使用该命令设置文件名模板, 示例: - /fnametmpl 图片_{{"{{.msgid}}"}}_{{"{{.msgdate}}"}}.jpg + /template 图片_{{"{{.msgid}}"}}_{{"{{.msgdate}}"}}.jpg 可用变量: - {{"{{.msgid}}"}}: 消息ID @@ -309,46 +311,52 @@ bot: - {{"{{.origname}}"}}: 媒体的原始文件名 (如果有) - {{"{{.chatid}}"}}: 消息的聊天ID - 模板仅在文件名策略设置为 '自定义模板' 时生效, - 且模板解析错误时会回退到默认文件名 + 模板仅在文件名策略为“自定义模板”时生效; + 解析错误时将回退到默认文件名 info_template_updated: "已更新文件名模板" info_current_template_prefix: "当前模板: {{.Template}}" - dl: - usage: "用法: /dl <链接1> <链接2> ..." - error_no_valid_links: "没有有效的链接可供下载" - info_files_select_storage: "共 {{.Count}} 个文件, 请选择存储位置" - ytdlp: - usage: "用法: /ytdlp [选项] [URL2] ...\n示例:\n /ytdlp https://example.com/video\n /ytdlp --format best https://example.com/video\n /ytdlp --extract-audio --audio-format mp3 https://example.com/video" - error_no_valid_urls: "没有有效的 URL" - info_urls_select_storage: "共 {{.Count}} 个链接, 请选择存储位置" - info_downloading: "正在通过 yt-dlp 下载..." - error_download_failed: "yt-dlp 下载失败: {{.Error}}" + download: + usage: | + 用法: /download <链接...> [yt-dlp 选项] + 可选择直接下载、Aria2 或视频解析。 + 示例: + /download https://example.com/file.zip + /download https://www.youtube.com/watch?v=... + /download --format best https://www.youtube.com/watch?v=... + prompt_select_engine: "找到 {{.Count}} 个链接,请选择下载方式" + button_direct: "直接下载" + button_aria2: "Aria2" + button_video: "视频(yt-dlp)" + info_select_storage: "请为 {{.Count}} 个链接选择存储位置" + error_no_valid_links: "未找到有效的下载链接" + error_no_engine: "无可用的下载方式" transfer: usage: | 用法: /transfer :/ [filter] 示例: /transfer local1:/downloads /transfer alist1:/media/photos - /transfer webdav1:/files ".*\.mp4$" + /transfer webdav1:/videos ".*\.mp4$" error_invalid_source: "源路径格式无效,应为: storage_name:/path" error_invalid_target: "目标路径格式无效,应为: storage_name:/path" - error_storage_not_found: "存储端 '{{.StorageName}}' 不存在或您无权访问: {{.Error}}" - error_storage_not_listable: "存储端 '{{.StorageName}}' 不支持列举文件功能" - error_storage_not_readable: "存储端 '{{.StorageName}}' 不支持读取文件功能" - error_target_not_found: "目标存储端 '{{.StorageName}}' 不存在或您无权访问: {{.Error}}" + error_storage_not_found: "存储端“{{.StorageName}}”不存在或无权访问: {{.Error}}" + error_storage_not_listable: "存储端“{{.StorageName}}”不支持列举文件" + error_storage_not_readable: "存储端“{{.StorageName}}”不支持读取文件" + error_target_not_found: "目标存储端“{{.StorageName}}”不存在或无权访问: {{.Error}}" info_fetching_files: "正在获取文件列表..." error_list_files_failed: "获取文件列表失败: {{.Error}}" error_invalid_regex: "正则表达式无效: {{.Error}}" - error_no_files_to_transfer: "目录中没有可传输的文件" + error_no_files_to_transfer: "目录中无待传输文件" error_add_task_failed: "添加任务失败: {{.Error}}" info_task_added: "已添加 {{.Count}} 个文件到传输队列\n总大小: {{.SizeMB}} MB\n任务 ID: {{.TaskID}}" start_stats: "总文件数: {{.Count}}\n总大小: {{.SizeMB}} MB" info_files_select_storage: "共 {{.Count}} 个文件 (总大小: {{.SizeMB}} MB),请选择目标存储位置" error_build_storage_select_keyboard_failed: "构建存储选择键盘失败: {{.Error}}" + retry: + info_started: "已创建重试任务: {{.TaskID}}({{.Count}} 个文件)" + error_failed: "重试任务失败: {{.Error}}" cancel: - usage: "用法: /cancel " error_cancel_failed: "取消任务失败: {{.Error}}" - info_cancel_requested: "已请求取消任务: {{.TaskID}}" info_cancelling_task: "正在取消任务..." media_group: info_saving_files: "正在保存文件..." @@ -365,13 +373,14 @@ bot: batch_item_transferring_unknown: "
↕️ {{.Index}}/{{.Total}} 传输中\n{{.Name}}\n速度:{{.Speed}}\n大小:{{.Current}} / 未知
" batch_item_uploading: "
⬆️ {{.Index}}/{{.Total}} 上传中\n{{.Name}}\n{{.Bar}} {{.Progress}}%\n速度:{{.Speed}}\n大小:{{.Current}} / {{.Size}}
" batch_item_retrying: "
🔁 {{.Index}}/{{.Total}} 上传重试\n{{.Name}}\n{{.Bar}} {{.Progress}}%\n重试次数:{{.Attempt}}/{{.Limit}}\n失败前速度:{{.Speed}}\n大小:{{.Current}} / {{.Size}}\n原因:{{.Reason}}
" - batch_item_confirming: "
⏳ {{.Index}}/{{.Total}} 等待中\n{{.Name}}\n{{.Bar}} {{.Progress}}%\n文件已上传,正在等待远端确认
" + batch_item_confirming: "
⏳ {{.Index}}/{{.Total}} 等待中\n{{.Name}}\n{{.Bar}} {{.Progress}}%\n已上传,等待远端确认
" batch_summary_hidden_active: "🔄 另有 {{.Count}} 个文件正在处理" batch_summary_confirming: "☁️ 已上传,等待整组发送:{{.Count}}" batch_summary_failed: "❌ 失败:{{.Count}}" batch_summary_skipped: "⏭️ 已跳过:{{.Count}}" batch_done: "✅ 处理完成\n\n文件数: {{.Count}}\n总大小: {{.Size}}" batch_done_with_skipped: "⚠️ 处理完成\n\n成功: {{.Success}}\n已跳过: {{.Skipped}}\n总大小: {{.Size}}" + batch_done_with_failed: "⚠️ 处理完成(部分失败)\n\n成功: {{.Success}}\n失败: {{.Failed}}\n已跳过: {{.Skipped}}\n总大小: {{.Size}}{{.List}}" batch_canceled: "🚫 任务已取消\n\n文件数: {{.Total}}\n已完成: {{.Completed}}\n未完成: {{.Incomplete}}\n已跳过: {{.Skipped}}" batch_failed_item: "❌ 处理失败\n\n失败文件: {{.Index}}. {{.Name}}\n失败阶段: {{.Stage}}\n失败进度: {{.Progress}}\n失败前速度: {{.Speed}}\n原因: {{.Reason}}\n\n✅ 已完成: {{.Completed}}\n❌ 失败: {{.Failed}}\n⏹️ 未完成: {{.Incomplete}}" batch_failed_group: "❌ 批量上传失败\n\n受影响文件: {{.Affected}} 个\n原因: {{.Reason}}\n\n✅ 已完成: {{.Completed}}\n❌ 批次失败: {{.Failed}}\n⏹️ 未完成: {{.Incomplete}}" @@ -441,4 +450,4 @@ bot: info_adding_aria2_download: "正在添加 Aria2 下载任务..." error_adding_aria2_download: "添加 Aria2 下载任务失败: {{.Error}}" info_aria2_download_added: "Aria2 下载任务已添加, GID: {{.GID}}" - info_select_storage: "请选择存储位置, 选择后将添加到 Aria2 下载队列" + info_select_storage: "请选择存储位置,确认后将加入 Aria2 下载队列" diff --git a/common/mediafilter/mediafilter.go b/common/mediafilter/mediafilter.go new file mode 100644 index 00000000..5d6147c2 --- /dev/null +++ b/common/mediafilter/mediafilter.go @@ -0,0 +1,36 @@ +// Package mediafilter applies the instance-wide allowed media type setting. +package mediafilter + +import ( + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/mediatype" +) + +// Allowed reports whether media should be processed under the current +// allowlist. An empty allowlist permits every supported media object. +func Allowed(media tg.MessageMediaClass) bool { + allowed := config.C().Media.AllowedMediaTypes + if len(allowed) == 0 { + return true + } + typ, ok := mediatype.Classify(media) + if !ok { + return false + } + return Allows(typ) +} + +// Allows reports whether a known media category is enabled. +func Allows(typ mediatype.Type) bool { + allowed := config.C().Media.AllowedMediaTypes + if len(allowed) == 0 { + return true + } + for _, value := range allowed { + if value == string(typ) { + return true + } + } + return false +} diff --git a/common/selfupdate/selfupdate.go b/common/selfupdate/selfupdate.go index 112f4c4c..63304d28 100644 --- a/common/selfupdate/selfupdate.go +++ b/common/selfupdate/selfupdate.go @@ -12,16 +12,25 @@ import ( "sync/atomic" "github.com/blang/semver" + "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/config" "github.com/unvgo/ghselfupdate" ) var ( - ErrDevBuild = errors.New("not a release build") - ErrDocker = errors.New("built into a Docker image") - ErrNotWritable = errors.New("the binary's folder is not writable") + ErrDevBuild = errors.New("not a release build") + ErrDocker = errors.New("built into a Docker image") + ErrNotWritable = errors.New("the binary's folder is not writable") + ErrRestart = errors.New(RestartSignal) + ErrNoRelease = errors.New("no release found") + ErrMajorUpgrade = errors.New("major upgrade requires a manual download and migration review") + ErrAlreadyLatest = errors.New("already on the latest version") ) +// RestartSignal marks a handler error that asks the client to restart the +// process after an update. +const RestartSignal = "saveanybot-restart" + var ( restarting atomic.Bool @@ -74,15 +83,39 @@ func Updatable() error { return nil } +// CheckTarget validates that latest can be installed by this build. +func CheckTarget(current, latest semver.Version) error { + if latest.Major > current.Major { + return fmt.Errorf("%w: %s -> %s", ErrMajorUpgrade, current, latest) + } + if !latest.GT(current) { + return fmt.Errorf("%w: %s", ErrAlreadyLatest, current) + } + return nil +} + func Update(current semver.Version) (*ghselfupdate.Release, error) { if err := Updatable(); err != nil { return nil, err } - rel, err := ghselfupdate.UpdateSelf(current, config.GitRepo) + latest, found, err := ghselfupdate.DetectLatest(config.GitRepo) if err != nil { + return nil, fmt.Errorf("failed to detect latest release: %w", err) + } + if !found { + return nil, ErrNoRelease + } + if err := CheckTarget(current, latest.Version); err != nil { + return nil, err + } + exe, err := SelfPath() + if err != nil { + return nil, err + } + if err := ghselfupdate.DefaultUpdater().UpdateTo(latest, exe); err != nil { return nil, fmt.Errorf("failed to update: %w", err) } - return rel, nil + return latest, nil } func Restart() error { @@ -96,6 +129,7 @@ func Restart() error { func CleanLeftovers() { exe, err := SelfPath() if err != nil { + log.Warnf("Failed to resolve executable path for cleanup: %v", err) return } cleanLeftovers(exe) @@ -103,10 +137,18 @@ func CleanLeftovers() { func cleanLeftovers(exe string) { dir, name := filepath.Split(exe) - _ = os.Remove(filepath.Join(dir, "."+name+".old")) + if err := os.Remove(filepath.Join(dir, "."+name+".old")); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Warnf("Failed to remove stale update file: %v", err) + } staged := filepath.Join(dir, "."+name+".new") - if same, err := sameContent(exe, staged); err == nil && same { - _ = os.Remove(staged) + if same, err := sameContent(exe, staged); err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Warnf("Failed to compare staged update: %v", err) + } + } else if same { + if err := os.Remove(staged); err != nil { + log.Warnf("Failed to remove staged update: %v", err) + } } } @@ -115,8 +157,14 @@ func writable(dir string) bool { if err != nil { return false } - f.Close() - os.Remove(f.Name()) + if err := f.Close(); err != nil { + log.Warnf("Failed to close update probe: %v", err) + return false + } + if err := os.Remove(f.Name()); err != nil { + log.Warnf("Failed to remove update probe: %v", err) + return false + } return true } diff --git a/common/tdler/dler.go b/common/tdler/dler.go index f965bcee..f73b7056 100644 --- a/common/tdler/dler.go +++ b/common/tdler/dler.go @@ -5,16 +5,41 @@ import ( "github.com/gotd/td/telegram/downloader" "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/client/relay" "github.com/krau/SaveAny-Bot/common/utils/dlutil" + "github.com/krau/SaveAny-Bot/common/utils/rpclimit" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/pkg/consts/tglimit" "github.com/krau/SaveAny-Bot/pkg/tfile" ) -func NewDownloader(file tfile.TGFile) *downloader.Builder { +func NewDownloader(ctx context.Context, file tfile.TGFile) (*downloader.Builder, func(context.Context)) { + file, cleanup := relay.Prepare(ctx, file) + return newDownloader(file), cleanup +} + +func newDownloader(file tfile.TGFile) *downloader.Builder { return downloader.NewDownloader().WithPartSize(tglimit.MaxPartSize). - Download(eofAwareClient{Client: file.Dler(), size: file.Size()}, file.Location()). - WithThreads(dlutil.BestThreads(file.Size(), config.C().Threads)) + Download(eofAwareClient{Client: rpclimit.WrapDownloader(file.Dler()), size: file.Size()}, file.Location()). + WithThreads(downloadThreads(file)) +} + +func downloadThreads(file tfile.TGFile) int { + cfg := config.C() + if cfg.DownloadThreads > 0 { + return cfg.DownloadThreads + } + threads := dlutil.BestThreads(file.Size(), cfg.Threads) + if scalable, ok := file.(interface{ WorkerCount() int }); ok { + // A relay file distributes chunks across helpers. Give each helper the + // same per-worker concurrency; the global transfer budget still caps the + // real number of in-flight RPCs in rpclimit. + threads *= max(scalable.WorkerCount(), 1) + } + if cfg.MaxDownloadThreads > 0 { + threads = min(threads, cfg.MaxDownloadThreads) + } + return max(threads, 1) } // eofAwareClient answers upload.getFile requests at or past the end of the diff --git a/common/tdler/download_cache.go b/common/tdler/download_cache.go new file mode 100644 index 00000000..dc17debc --- /dev/null +++ b/common/tdler/download_cache.go @@ -0,0 +1,125 @@ +package tdler + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/charmbracelet/log" + + "github.com/krau/SaveAny-Bot/client/relay" + "github.com/krau/SaveAny-Bot/pkg/tfile" +) + +// DownloadToCache downloads file into cachePath, reporting download progress +// through wrap. A complete cache file is reused as-is (e.g. when the previous +// run was interrupted during upload). Downloads with a known size resume from +// a partial .part file tracked by a resume bitmap, which is removed once the +// download is finalized. Files with an unknown size (e.g. photos) are fetched +// with the plain downloader. +func DownloadToCache( + ctx context.Context, + file tfile.TGFile, + cachePath string, + wrap func(io.WriterAt) io.WriterAt, +) error { + if cachePath == "" { + return fmt.Errorf("empty cache path") + } + logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", file.Name())) + if file.Size() > 0 { + if stat, err := os.Stat(cachePath); err == nil && stat.Size() == file.Size() { + logger.Info("Cache file already complete, skipping download") + if err := os.Remove(cachePath + ".part"); err != nil && !errors.Is(err, os.ErrNotExist) { + logger.Warnf("Failed to remove stale part file: %v", err) + } + relay.CleanupResumeState(ctx, relay.ResumeStatePath(cachePath)) + if err := RemoveResumeState(ResumeStatePath(cachePath + ".part")); err != nil { + logger.Warnf("Failed to remove stale resume state: %v", err) + } + return nil + } + } + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + var cleanup func(context.Context) + if file.Size() > 0 { + file, cleanup = relay.PrepareResumable(ctx, file, relay.ResumeStatePath(cachePath)) + } else { + file, cleanup = relay.Prepare(ctx, file) + } + if file.Size() <= 0 { + if cleanup != nil { + defer cleanup(ctx) + } + // Unknown size (e.g. photos) cannot be resumed; use the plain downloader. + localFile, err := os.Create(cachePath) + if err != nil { + return fmt.Errorf("failed to create cache file: %w", err) + } + defer func() { + if err := localFile.Close(); err != nil { + logger.Warnf("Failed to close cache file: %v", err) + } + }() + if _, err := newDownloader(file).Parallel(ctx, wrap(localFile)); err != nil { + return fmt.Errorf("download failed: %w", err) + } + return nil + } + partPath := cachePath + ".part" + // 不截断已存在的 .part: 位图标记的已完成块依赖既有字节。 + partFile, err := os.OpenFile(partPath, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return fmt.Errorf("failed to open cache file: %w", err) + } + downloadErr := DownloadResumable( + ctx, file, wrap(partFile), + downloadThreads(file), + ResumeStatePath(partPath), + ) + if downloadErr != nil { + // Keep the relay message and sidecar so a later run can reuse them. + return errors.Join(downloadErr, partFile.Close()) + } + if cleanup != nil { + defer cleanup(ctx) + } + // 落盘后再关闭并 rename: 确保位图标记的字节确实持久, 掉电后不会 + // 出现 "位图完整但数据页丢失" 的零字节洞。 + if err := partFile.Sync(); err != nil { + return errors.Join(fmt.Errorf("failed to sync cache file: %w", err), partFile.Close()) + } + closeErr := partFile.Close() + if closeErr != nil { + return fmt.Errorf("failed to close cache file: %w", closeErr) + } + stat, err := os.Stat(partPath) + if err != nil { + return fmt.Errorf("failed to stat downloaded file: %w", err) + } + if stat.Size() > file.Size() { + if err := os.Truncate(partPath, file.Size()); err != nil { + return fmt.Errorf("truncate downloaded file: %w", err) + } + stat, err = os.Stat(partPath) + if err != nil { + return fmt.Errorf("failed to stat downloaded file: %w", err) + } + } + if stat.Size() != file.Size() { + return fmt.Errorf("downloaded size %d does not match expected %d", stat.Size(), file.Size()) + } + if err := os.Rename(partPath, cachePath); err != nil { + return fmt.Errorf("failed to finalize download: %w", err) + } + // 清理位图是尽力而为: 下载已完成, 清理失败不应使任务失败。 + if err := RemoveResumeState(ResumeStatePath(partPath)); err != nil { + logger.Warnf("Failed to remove resume state: %v", err) + } + return nil +} diff --git a/common/tdler/resume.go b/common/tdler/resume.go new file mode 100644 index 00000000..9ff2e4a1 --- /dev/null +++ b/common/tdler/resume.go @@ -0,0 +1,454 @@ +package tdler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/bits" + "net" + "os" + "strings" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/gotd/td/telegram/downloader" + "github.com/gotd/td/tg" + "github.com/gotd/td/tgerr" + "golang.org/x/sync/errgroup" + + "github.com/krau/SaveAny-Bot/common/utils/rpclimit" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/pkg/consts/tglimit" + "github.com/krau/SaveAny-Bot/pkg/tfile" +) + +const maxChunkRetries = 20 + +// chunkRetryLimit returns the configured timeout retry limit for one chunk. +func chunkRetryLimit() int { + limit := config.C().ChunkRetries + if limit < 1 { + return maxChunkRetries + } + return limit +} + +// errCDNRedirect reports a chunk served from a CDN. Raw upload.getFile +// requests cannot follow CDN redirects (they need gotd's re-hash dance), so +// the download falls back to the plain downloader, which handles them. +var errCDNRedirect = errors.New("CDN redirect") + +func partPathOf(bitmapPath string) string { + return strings.TrimSuffix(bitmapPath, ".bitmap") +} + +// resumeBitmap records which partSize-aligned blocks of a download have been +// durably written, so an interrupted download can continue after a restart. +// The bitmap file is rewritten atomically after every completed block. +type resumeBitmap struct { + PartSize int `json:"part_size"` + Size int64 `json:"size"` + Blocks []uint64 `json:"blocks"` + SourceKind string `json:"source_kind,omitempty"` + SourceID int64 `json:"source_id,omitempty"` + SourceAccessHash int64 `json:"source_access_hash,omitempty"` + + mu sync.Mutex + dirty int + lastSave time.Time +} + +func newResumeBitmap(size int64, identity tfile.FileIdentity) *resumeBitmap { + b := &resumeBitmap{ + PartSize: tglimit.MaxPartSize, + Size: size, + SourceKind: identity.Kind, + SourceID: identity.ID, + SourceAccessHash: identity.AccessHash, + } + b.ensureBlocks() + return b +} + +func (b *resumeBitmap) matches(identity tfile.FileIdentity) bool { + if b.SourceKind == "" && b.SourceID == 0 && b.SourceAccessHash == 0 { + // State written by an earlier version without identity metadata. + return true + } + return b.SourceKind == identity.Kind && b.SourceID == identity.ID && b.SourceAccessHash == identity.AccessHash +} + +func (b *resumeBitmap) ensureBlocks() { + if need := (b.blockCount() + 63) / 64; len(b.Blocks) < need { + b.Blocks = make([]uint64, need) + } +} + +func (b *resumeBitmap) blockCount() int { + return int((b.Size + int64(b.PartSize) - 1) / int64(b.PartSize)) +} + +func (b *resumeBitmap) isDone(block int) bool { + return b.Blocks[block/64]&(1< highest { + highest = block + } + } + if highest < 0 { + return 0 + } + covered := int64(highest+1) * int64(b.PartSize) + return min(covered, b.Size) +} + +// maxResumableSize bounds the bitmap allocation derived from a (possibly +// corrupt) size field: 1 TiB needs ~16 KiB of bitmap words, far above any +// real Telegram file. +const maxResumableSize int64 = 1 << 40 + +func loadResumeBitmap(path string) (*resumeBitmap, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read resume bitmap: %w", err) + } + var b resumeBitmap + if err := json.Unmarshal(data, &b); err != nil { + // 无法解析的位图 (外部损坏): 删除并视为不存在, 全量重下自愈。 + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, fmt.Errorf("remove invalid resume bitmap: %w", removeErr) + } + return nil, nil + } + // 位图只写 PartSize=MaxPartSize; 其他值或超大 size 都是损坏数据, + // 在按 size 分配内存前拒绝, 防止构造的位图触发巨量分配。 + if b.PartSize != tglimit.MaxPartSize || b.Size > maxResumableSize { + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return nil, fmt.Errorf("remove incompatible resume bitmap: %w", removeErr) + } + return nil, nil + } + b.ensureBlocks() + return &b, nil +} + +func (b *resumeBitmap) save(path string) error { + b.mu.Lock() + defer b.mu.Unlock() + return b.saveLocked(path) +} + +func (b *resumeBitmap) saveLocked(path string) error { + data, err := json.Marshal(b) + if err != nil { + return fmt.Errorf("marshal resume bitmap: %w", err) + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("write resume bitmap: %w", err) + } + return os.Rename(tmp, path) +} + +func (b *resumeBitmap) markAndSave(block int, path string) error { + b.mu.Lock() + defer b.mu.Unlock() + b.markDone(block) + b.dirty++ + // Persist at most once per second or every 16 completed blocks. A crash + // between saves only re-downloads the unwritten tail; it cannot mark + // missing data as complete. + if b.dirty < 16 && !b.lastSave.IsZero() && time.Since(b.lastSave) < time.Second { + return nil + } + if err := b.saveLocked(path); err != nil { + return err + } + b.dirty = 0 + b.lastSave = time.Now() + return nil +} + +func (b *resumeBitmap) flush(path string) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.dirty == 0 { + return nil + } + if err := b.saveLocked(path); err != nil { + return err + } + b.dirty = 0 + b.lastSave = time.Now() + return nil +} + +func isRetryableChunkError(ctx context.Context, err error) bool { + if err == nil || ctx.Err() != nil { + return false + } + if tgerr.Is(err, tg.ErrTimeout) || + errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) && (netErr.Timeout() || netErr.Temporary()) { + return true + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, net.ErrClosed) { + return true + } + // gotd's RPC engine closes in-flight requests when it reconnects. The + // wrapped engine error has no exported type, so match its stable text. + message := strings.ToLower(err.Error()) + for _, fragment := range []string{ + "engine forcibly closed", + "connection reset by peer", + "connection closed", + "broken pipe", + "retryuntilack", + "retry limit reached", + "use of closed network connection", + } { + if strings.Contains(message, fragment) { + return true + } + } + return false +} + +func waitChunkRetry(ctx context.Context, attempt int) error { + delay := 100 * time.Millisecond + if attempt > 1 { + delay = min(100*time.Millisecond< limit { + return nil, fmt.Errorf("get chunk at %d: server returned %d bytes, limit %d", offset, len(r.Bytes), limit) + } + return r.Bytes, nil + case *tg.UploadFileCDNRedirect: + return nil, fmt.Errorf("%w (dc %d)", errCDNRedirect, r.DCID) + default: + return nil, fmt.Errorf("unexpected upload.getFile response %T", res) + } + } + if flood, ferr := tgerr.FloodWait(ctx, err); ferr != nil { + if flood { + // FloodWait already slept; retry. + continue + } + if isRetryableChunkError(ctx, ferr) { + retries++ + if retries >= retryLimit { + return nil, fmt.Errorf("get chunk at %d: retry limit reached: %w", offset, ferr) + } + log.FromContext(ctx).Debugf( + "Retrying chunk at %d after transient error (%d/%d): %v", + offset, retries, retryLimit, ferr, + ) + if werr := waitChunkRetry(ctx, retries); werr != nil { + return nil, fmt.Errorf("get chunk at %d: %w", offset, werr) + } + continue + } + return nil, fmt.Errorf("get chunk at %d: %w", offset, ferr) + } + } +} + +// DownloadResumable downloads file to w in partSize chunks, skipping blocks +// already recorded as complete in bitmapPath and persisting every completed +// block so an interrupted download can resume. A missing or incompatible +// bitmap starts a full download. Requires a known, non-zero file size. +func DownloadResumable( + ctx context.Context, + file tfile.TGFile, + w io.WriterAt, + threads int, + bitmapPath string, +) error { + if file.Size() <= 0 { + return fmt.Errorf("resumable download requires a known size") + } + if threads < 1 { + threads = 1 + } + bm, err := loadResumeBitmap(bitmapPath) + if err != nil { + return err + } + if bm != nil && !bm.matches(tfile.Identity(file.Location())) { + if err := os.Remove(bitmapPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale resume bitmap: %w", err) + } + bm = nil + } + // 位图描述的数据文件 (bitmapPath 去掉 .bitmap 后缀) 必须存在且非空, + // 且至少覆盖位图标记为完成的最大块范围: 若缺失、为空或更短, + // 已标记完成的块字节已丢失, 必须重置位图全量重下 (否则 WriteAt 会 + // 在文件中留下零字节洞, 而最终尺寸校验恰好通过, 造成静默损坏)。 + if bm != nil { + if stat, err := os.Stat(partPathOf(bitmapPath)); err != nil || stat.Size() < bm.coveredBytes() { + if err := os.Remove(bitmapPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("reset stale resume bitmap: %w", err) + } + bm = nil + } + } + if bm == nil || bm.PartSize != tglimit.MaxPartSize || bm.Size != file.Size() { + // 位图缺失或不兼容: 截断旧的部分文件, 避免残留尾部字节导致最终大小校验失败。 + if err := os.Truncate(partPathOf(bitmapPath), 0); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("truncate stale part file: %w", err) + } + bm = newResumeBitmap(file.Size(), tfile.Identity(file.Location())) + if err := bm.save(bitmapPath); err != nil { + return err + } + } + missing := bm.missingBlocks() + if len(missing) == 0 { + return nil + } + + client := rpclimit.WrapDownloader(file.Dler()) + location := file.Location() + var writeMu sync.Mutex + eg, gctx := errgroup.WithContext(ctx) + eg.SetLimit(threads) + for _, block := range missing { + if err := gctx.Err(); err != nil { + break + } + eg.Go(func() error { + offset := int64(block) * int64(bm.PartSize) + data, err := fetchChunk(gctx, client, location, offset, bm.PartSize) + if err != nil { + return err + } + want := int64(bm.PartSize) + if offset+want > bm.Size { + want = bm.Size - offset + } + if int64(len(data)) != want { + return fmt.Errorf("chunk at offset %d has size %d, want %d", offset, len(data), want) + } + writeMu.Lock() + defer writeMu.Unlock() + n, err := w.WriteAt(data, offset) + if err != nil { + return fmt.Errorf("write chunk at offset %d: %w", offset, err) + } + if n != len(data) { + return fmt.Errorf("write chunk at offset %d: short write %d/%d", offset, n, len(data)) + } + return bm.markAndSave(block, bitmapPath) + }) + } + err = eg.Wait() + if flushErr := bm.flush(bitmapPath); flushErr != nil { + return flushErr + } + if err != nil { + if errors.Is(err, errCDNRedirect) { + // CDN 托管的文件无法用裸 upload.getFile 续传, 回退到 gotd 下载器 + // (它内部处理 CDN 重定向), 全量重下本文件。 + log.FromContext(ctx).Warn("File served from CDN, falling back to plain downloader") + if _, err := newDownloader(file).Parallel(ctx, w); err != nil { + return fmt.Errorf("plain download after CDN redirect: %w", err) + } + return nil + } + return err + } + if !bm.complete() { + return fmt.Errorf("download finished with missing blocks") + } + return nil +} + +// RemoveResumeState deletes the bitmap file of a completed download. +func RemoveResumeState(bitmapPath string) error { + if err := os.Remove(bitmapPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove resume bitmap: %w", err) + } + if err := os.Remove(bitmapPath + ".tmp"); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove resume bitmap temp: %w", err) + } + return nil +} + +// ResumeStatePath returns the bitmap path for a download cache file. +func ResumeStatePath(cachePath string) string { + return cachePath + ".bitmap" +} diff --git a/common/tdler/resume_identity_test.go b/common/tdler/resume_identity_test.go new file mode 100644 index 00000000..3e7b8932 --- /dev/null +++ b/common/tdler/resume_identity_test.go @@ -0,0 +1,23 @@ +package tdler + +import ( + "testing" + + "github.com/krau/SaveAny-Bot/pkg/tfile" +) + +func TestResumeBitmapFileIdentity(t *testing.T) { + a := tfile.FileIdentity{Kind: "document", ID: 1, AccessHash: 2} + b := tfile.FileIdentity{Kind: "document", ID: 3, AccessHash: 2} + bm := newResumeBitmap(2*1024*1024, a) + if !bm.matches(a) { + t.Fatal("bitmap did not match its source file") + } + if bm.matches(b) { + t.Fatal("bitmap matched a different source file") + } + legacy := &resumeBitmap{PartSize: 1024, Size: 1} + if !legacy.matches(b) { + t.Fatal("legacy bitmap without identity should be accepted") + } +} diff --git a/common/upload/upload.go b/common/upload/upload.go new file mode 100644 index 00000000..95d61c53 --- /dev/null +++ b/common/upload/upload.go @@ -0,0 +1,115 @@ +// Package upload provides the shared local-file upload path used by the CLI +// commands. It keeps retries, cancellation, and confirmed progress consistent +// across one-shot and watch uploads. +package upload + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/krau/SaveAny-Bot/common/utils/ioutil" + "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" + "github.com/krau/SaveAny-Bot/storage" +) + +type Options struct { + Storage storage.Storage + Overwrite bool + Retries int + RetryDelay time.Duration + OnProgress func(uploaded, total int64) +} + +func (o Options) validate() error { + if o.Storage == nil { + return errors.New("upload: storage is required") + } + if o.Retries < 1 { + return errors.New("upload: retries must be at least 1") + } + if o.RetryDelay < 0 { + return errors.New("upload: retry delay must not be negative") + } + return nil +} + +// SaveFile uploads a local file, rewinding it for every retry. +func SaveFile(ctx context.Context, localPath, storagePath string, opts Options) (retErr error) { + if err := opts.validate(); err != nil { + return err + } + file, err := os.Open(localPath) + if err != nil { + return fmt.Errorf("upload: open %s: %w", localPath, err) + } + defer func() { + retErr = errors.Join(retErr, file.Close()) + }() + + info, err := file.Stat() + if err != nil { + return fmt.Errorf("upload: stat %s: %w", localPath, err) + } + if info.IsDir() { + return fmt.Errorf("upload: %s is a directory", localPath) + } + return Save(ctx, file, info.Size(), storagePath, opts) +} + +// Save uploads a seekable reader. The reader is rewound before each attempt so +// a retry never sends a partially consumed stream. +func Save(ctx context.Context, reader io.ReadSeeker, size int64, storagePath string, opts Options) error { + if err := opts.validate(); err != nil { + return err + } + uploadCtx := context.WithValue(ctx, ctxkey.ContentLength, size) + if opts.Overwrite { + uploadCtx = storage.WithOverwrite(uploadCtx) + } + + var lastErr error + for attempt := 0; attempt < opts.Retries; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("upload: rewind: %w", err) + } + err := saveOnce(uploadCtx, reader, size, storagePath, opts) + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + lastErr = err + if attempt+1 >= opts.Retries { + break + } + if opts.RetryDelay > 0 { + timer := time.NewTimer(opts.RetryDelay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + } + return fmt.Errorf("upload failed after %d attempt(s): %w", opts.Retries, lastErr) +} + +func saveOnce(ctx context.Context, reader io.ReadSeeker, size int64, storagePath string, opts Options) error { + if progressSaver, ok := opts.Storage.(storage.StorageProgressSaver); ok { + return progressSaver.SaveWithProgress(ctx, reader, storagePath, opts.OnProgress) + } + if opts.OnProgress == nil { + return opts.Storage.Save(ctx, reader, storagePath) + } + progressReader := ioutil.NewProgressReader(reader, size, opts.OnProgress) + return opts.Storage.Save(ctx, progressReader, storagePath) +} diff --git a/common/upload/upload_test.go b/common/upload/upload_test.go new file mode 100644 index 00000000..bc342973 --- /dev/null +++ b/common/upload/upload_test.go @@ -0,0 +1,55 @@ +package upload + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/krau/SaveAny-Bot/config/storage" + storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" +) + +func TestSaveRetriesAndRewinds(t *testing.T) { + stor := &retryStorage{failures: 1} + err := Save(context.Background(), strings.NewReader("payload"), 7, "file.txt", Options{ + Storage: stor, + Retries: 2, + RetryDelay: time.Millisecond, + }) + if err != nil { + t.Fatalf("Save: %v", err) + } + if stor.attempts != 2 { + t.Fatalf("attempts = %d, want 2", stor.attempts) + } + if string(stor.data) != "payload" { + t.Fatalf("uploaded data = %q, want payload", stor.data) + } +} + +type retryStorage struct { + attempts int + failures int + data []byte +} + +func (s *retryStorage) Init(context.Context, storage.StorageConfig) error { return nil } +func (s *retryStorage) Type() storenum.StorageType { return storenum.Local } +func (s *retryStorage) Name() string { return "test" } +func (s *retryStorage) Exists(context.Context, string) (bool, error) { return false, nil } + +func (s *retryStorage) Save(_ context.Context, r io.Reader, _ string) error { + s.attempts++ + if s.attempts <= s.failures { + return errors.New("transient failure") + } + data, err := io.ReadAll(r) + if err != nil { + return err + } + s.data = data + return nil +} diff --git a/common/utils/fsutil/fs.go b/common/utils/fsutil/fs.go index 0295d081..c1b1a18c 100644 --- a/common/utils/fsutil/fs.go +++ b/common/utils/fsutil/fs.go @@ -3,6 +3,7 @@ package fsutil import ( "errors" "os" + "path" "path/filepath" "strings" "unicode" @@ -60,6 +61,18 @@ func CreateFile(fp string) (*File, error) { return &File{File: file}, nil } +// SafeBaseName reduces a remote-provided name to one safe filename component. +func SafeBaseName(name string) string { + name = strings.ReplaceAll(name, "\\", "/") + name = path.Base(name) + name = NormalizePathname(name) + switch name { + case "", ".", "..": + return "file" + } + return name +} + func NormalizePathname(s string) string { specials := `\/:*?"<>|` + "\n\r\t" var builder strings.Builder diff --git a/common/utils/fsutil/unique.go b/common/utils/fsutil/unique.go index f8b98d7c..b817d9a6 100644 --- a/common/utils/fsutil/unique.go +++ b/common/utils/fsutil/unique.go @@ -9,19 +9,28 @@ import ( ) // UniquePath returns a non-taken path under basePath: the name itself, then -// numbered variants, then a random suffix. -func UniquePath(basePath, name string, exists func(candidate string) bool, maxAttempts int) string { +// numbered variants, then a random suffix. The exists callback may return an +// error; callers must not treat a failed check as "not taken". +func UniquePath(basePath, name string, exists func(candidate string) (bool, error), maxAttempts int) (string, error) { candidate := path.Join(basePath, name) - if !exists(candidate) { - return candidate + taken, err := exists(candidate) + if err != nil { + return "", err + } + if !taken { + return candidate, nil } ext := path.Ext(name) stem := strings.TrimSuffix(name, ext) for i := 1; i <= maxAttempts; i++ { candidate = path.Join(basePath, fmt.Sprintf("%s_%d%s", stem, i, ext)) - if !exists(candidate) { - return candidate + taken, err = exists(candidate) + if err != nil { + return "", err + } + if !taken { + return candidate, nil } } - return path.Join(basePath, fmt.Sprintf("%s_%s%s", stem, xid.New().String(), ext)) + return path.Join(basePath, fmt.Sprintf("%s_%s%s", stem, xid.New().String(), ext)), nil } diff --git a/common/utils/netutil/proxy.go b/common/utils/netutil/proxy.go index 23cd95c7..b7eee094 100644 --- a/common/utils/netutil/proxy.go +++ b/common/utils/netutil/proxy.go @@ -14,65 +14,83 @@ import ( "golang.org/x/net/proxy" ) -func NewProxyHTTPClient(proxyUrl string) (*http.Client, error) { - if proxyUrl == "" { - return http.DefaultClient, nil - } - transport, err := NewProxyTransport(proxyUrl) +func NewProxyHTTPClient(proxyURL string) (*http.Client, error) { + transport, err := NewProxyTransport(proxyURL) if err != nil { return nil, err } - return &http.Client{ - Transport: transport, - }, nil + return &http.Client{Transport: transport}, nil } -var ( - defaultProxyHttpClient *http.Client - onceLoadDefaultProxyHttpClient sync.Once -) +var defaultParserClient = struct { + sync.Mutex + key string + client *http.Client +}{} +// DefaultParserHTTPClient returns a parser client for the current parser proxy +// setting. The client is rebuilt when the setting changes so a settings reload +// does not leave requests pinned to the old proxy. func DefaultParserHTTPClient() *http.Client { - onceLoadDefaultProxyHttpClient.Do(func() { - client, err := NewProxyHTTPClient(config.C().Parser.Proxy) - if err != nil { - log.Warn("Failed to create default proxy HTTP client, using http.DefaultClient", "error", err) - defaultProxyHttpClient = http.DefaultClient - } else { - defaultProxyHttpClient = client + key := config.C().Parser.Proxy + defaultParserClient.Lock() + defer defaultParserClient.Unlock() + if defaultParserClient.client != nil && defaultParserClient.key == key { + return defaultParserClient.client + } + client, err := NewProxyHTTPClient(key) + if err != nil { + log.Warn("Failed to create default proxy HTTP client, using direct client", "error", err) + fallback, fallbackErr := NewProxyHTTPClient("") + if fallbackErr != nil { + log.Errorf("Failed to create direct parser HTTP client: %v", fallbackErr) + fallback = http.DefaultClient } - }) - return defaultProxyHttpClient + client = fallback + } + if defaultParserClient.client != nil { + if transport, ok := defaultParserClient.client.Transport.(*http.Transport); ok { + transport.CloseIdleConnections() + } + } + defaultParserClient.client = client + defaultParserClient.key = key + return client } func NewProxyTransport(proxyStr string) (*http.Transport, error) { - proxyURL, err := url.Parse(proxyStr) - if err != nil { - return nil, err - } transport := &http.Transport{ ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + } + if proxyStr == "" { + return transport, nil + } + proxyURL, err := url.Parse(proxyStr) + if err != nil { + return nil, err } switch proxyURL.Scheme { case "http", "https": transport.Proxy = http.ProxyURL(proxyURL) - case "socks5", "socks5h": dialer, err := proxy.FromURL(proxyURL, proxy.Direct) if err != nil { return nil, err } + contextDialer, ok := dialer.(proxy.ContextDialer) + if !ok { + return nil, fmt.Errorf("proxy %s does not support context cancellation", proxyURL.Scheme) + } transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - return dialer.(proxy.ContextDialer).DialContext(ctx, network, addr) + return contextDialer.DialContext(ctx, network, addr) } - default: return nil, fmt.Errorf("unsupported proxy type: %s", proxyURL.Scheme) } - return transport, nil } diff --git a/common/utils/netutil/safehttp.go b/common/utils/netutil/safehttp.go new file mode 100644 index 00000000..fb055f94 --- /dev/null +++ b/common/utils/netutil/safehttp.go @@ -0,0 +1,60 @@ +package netutil + +import ( + "context" + "fmt" + "net" + "net/http" + "time" +) + +// PublicOnlyHTTPClient returns a client that refuses to connect to loopback, +// private, link-local, or unspecified addresses. Use it for URLs taken from +// untrusted content. +func PublicOnlyHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + DialContext: publicDialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + }, + } +} + +func publicDialContext(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("netutil: resolve %s: %w", host, err) + } + dialer := &net.Dialer{Timeout: 10 * time.Second} + var lastErr error + for _, address := range addresses { + if !isPublicIP(address.IP) { + continue + } + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(address.IP.String(), port)) + if err == nil { + return conn, nil + } + lastErr = err + } + if lastErr != nil { + return nil, fmt.Errorf("netutil: no reachable public address for %s: %w", host, lastErr) + } + return nil, fmt.Errorf("netutil: %s resolves only to non-public addresses", host) +} + +func isPublicIP(ip net.IP) bool { + return ip.IsGlobalUnicast() && + !ip.IsPrivate() && + !ip.IsLoopback() && + !ip.IsLinkLocalUnicast() && + !ip.IsLinkLocalMulticast() && + !ip.IsUnspecified() +} diff --git a/common/utils/rpclimit/rpclimit.go b/common/utils/rpclimit/rpclimit.go new file mode 100644 index 00000000..c744d0db --- /dev/null +++ b/common/utils/rpclimit/rpclimit.go @@ -0,0 +1,126 @@ +// Package rpclimit bounds the number of concurrent Telegram file-part RPCs. +// +// Per-file threads alone do not bound concurrency: every task worker can run +// several elements, and each element opens its own set of part requests, so +// the real load is workers × elements × threads. Downloads and uploads share +// one budget derived from the configuration, which is re-read on every +// acquire so runtime setting changes apply without a restart. +package rpclimit + +import ( + "context" + "sync" + + "github.com/gotd/td/telegram/downloader" + "github.com/gotd/td/telegram/uploader" + "github.com/gotd/td/tg" + "github.com/krau/SaveAny-Bot/config" +) + +var transfer = &limiter{notify: make(chan struct{}, 1)} + +// Acquire reserves one transfer slot. It blocks until a slot is free or ctx is +// done. Every successful Acquire must be paired with Release. +func Acquire(ctx context.Context) error { + return transfer.acquire(ctx, config.TransferBudget()) +} + +// Release returns a previously acquired slot. +func Release() { + transfer.release() +} + +// WrapDownloader limits UploadGetFile calls made through the returned client. +func WrapDownloader(client downloader.Client) downloader.Client { + if client == nil { + return nil + } + return limitedDownloader{Client: client} +} + +// WrapUploader limits save-part calls made through the returned uploader +// client. +func WrapUploader(client uploader.Client) uploader.Client { + if client == nil { + return nil + } + return limitedUploader{Client: client} +} + +type limitedDownloader struct { + downloader.Client +} + +func (c limitedDownloader) UploadGetFile(ctx context.Context, req *tg.UploadGetFileRequest) (tg.UploadFileClass, error) { + if err := Acquire(ctx); err != nil { + return nil, err + } + defer Release() + return c.Client.UploadGetFile(ctx, req) +} + +type limitedUploader struct { + uploader.Client +} + +func (c limitedUploader) UploadSaveFilePart(ctx context.Context, req *tg.UploadSaveFilePartRequest) (bool, error) { + if err := Acquire(ctx); err != nil { + return false, err + } + defer Release() + return c.Client.UploadSaveFilePart(ctx, req) +} + +func (c limitedUploader) UploadSaveBigFilePart(ctx context.Context, req *tg.UploadSaveBigFilePartRequest) (bool, error) { + if err := Acquire(ctx); err != nil { + return false, err + } + defer Release() + return c.Client.UploadSaveBigFilePart(ctx, req) +} + +// limiter is a context-aware counting semaphore with a dynamic limit. +type limiter struct { + mu sync.Mutex + inFlight int + notify chan struct{} +} + +func (l *limiter) acquire(ctx context.Context, limit int) error { + limit = max(limit, 1) + for { + l.mu.Lock() + if l.inFlight < limit { + l.inFlight++ + // Hand one wake-up to another waiter when capacity remains. + if l.inFlight < limit { + l.signalLocked() + } + l.mu.Unlock() + return nil + } + l.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-l.notify: + } + } +} + +func (l *limiter) release() { + l.mu.Lock() + if l.inFlight > 0 { + l.inFlight-- + } + l.signalLocked() + l.mu.Unlock() +} + +// signalLocked wakes at most one waiter. +func (l *limiter) signalLocked() { + select { + case l.notify <- struct{}{}: + default: + } +} diff --git a/common/utils/strutil/string_test.go b/common/utils/strutil/string_test.go index 99c096a3..a751c6fe 100644 --- a/common/utils/strutil/string_test.go +++ b/common/utils/strutil/string_test.go @@ -107,13 +107,13 @@ func TestParseArgsRespectQuotes(t *testing.T) { }{ { name: "simple split", - input: `/rule add FILENAME-REGEX (?i)\.(mp4|mkv)$ "我的 Alist" /视频`, - want: []string{"/rule", "add", "FILENAME-REGEX", "(?i)\\.(mp4|mkv)$", "我的 Alist", "/视频"}, + input: `/rules add FILENAME-REGEX (?i)\.(mp4|mkv)$ "我的 Alist" /视频`, + want: []string{"/rules", "add", "FILENAME-REGEX", "(?i)\\.(mp4|mkv)$", "我的 Alist", "/视频"}, }, { name: "escaped quotes", - input: `/rule add "My \"Awesome\" Folder"`, - want: []string{"/rule", "add", `My "Awesome" Folder`}, + input: `/rules add "My \"Awesome\" Folder"`, + want: []string{"/rules", "add", `My "Awesome" Folder`}, }, { name: "escaped backslash", diff --git a/common/utils/tgutil/chatinfo.go b/common/utils/tgutil/chatinfo.go new file mode 100644 index 00000000..cd0eed7f --- /dev/null +++ b/common/utils/tgutil/chatinfo.go @@ -0,0 +1,140 @@ +package tgutil + +import ( + "sync" + + "github.com/charmbracelet/log" + "github.com/gotd/td/constant" + "github.com/gotd/td/tg" + "github.com/krau/mygotg/ext" +) + +// ChatInfo is the normalized display identity of a Telegram peer. +type ChatInfo struct { + ID int64 `json:"id"` + Title string `json:"title,omitempty"` + Username string `json:"username,omitempty"` +} + +var chatInfoCache sync.Map // map[int64]ChatInfo + +// ChatInfoFromExt resolves a peer's title and username from update entities, +// then from the peer cache/API. Empty fields are left empty. +func ChatInfoFromExt(ctx *ext.Context, peer tg.PeerClass) ChatInfo { + id := ChatIdFromPeer(peer) + if id == 0 { + return ChatInfo{} + } + if cached, ok := chatInfoCache.Load(id); ok { + return cached.(ChatInfo) + } + info := ChatInfo{ID: id} + if ctx != nil { + plain := constant.TDLibPeerID(id).ToPlain() + if ctx.Entities != nil { + switch { + case ctx.Entities.Channels[plain] != nil: + channel := ctx.Entities.Channels[plain] + info.Title, info.Username = channel.Title, channel.Username + case ctx.Entities.Chats[plain] != nil: + info.Title = ctx.Entities.Chats[plain].Title + case ctx.Entities.Users[plain] != nil: + user := ctx.Entities.Users[plain] + info.Title, info.Username = displayName(user), user.Username + } + } + if info.Title == "" && ctx.Raw != nil { + info.Title, info.Username = fetchChatInfo(ctx, peer, plain) + } + } + if info.Title != "" || info.Username != "" { + chatInfoCache.Store(id, info) + } + return info +} + +func displayName(user *tg.User) string { + if user == nil { + return "" + } + if user.LastName != "" { + return user.FirstName + " " + user.LastName + } + return user.FirstName +} + +func fetchChatInfo(ctx *ext.Context, peer tg.PeerClass, plainID int64) (string, string) { + switch peer.(type) { + case *tg.PeerChannel: + return fetchChannelInfo(ctx, plainID) + case *tg.PeerChat: + return fetchGroupInfo(ctx, plainID) + case *tg.PeerUser: + return fetchUserInfo(ctx, plainID) + } + return "", "" +} + +func fetchChannelInfo(ctx *ext.Context, channelID int64) (string, string) { + peer, err := ctx.ResolveInputPeerById(MarkedChannelID(channelID)) + if err != nil { + log.Debugf("Failed to resolve channel %d: %v", channelID, err) + return "", "" + } + channelPeer, ok := peer.(*tg.InputPeerChannel) + if !ok { + return "", "" + } + result, err := ctx.Raw.ChannelsGetChannels(ctx, []tg.InputChannelClass{ + &tg.InputChannel{ChannelID: channelID, AccessHash: channelPeer.AccessHash}, + }) + if err != nil { + log.Debugf("Failed to fetch channel %d: %v", channelID, err) + return "", "" + } + for _, item := range result.GetChats() { + if channel, ok := item.(*tg.Channel); ok && channel.ID == channelID { + return channel.Title, channel.Username + } + } + return "", "" +} + +func fetchGroupInfo(ctx *ext.Context, chatID int64) (string, string) { + result, err := ctx.Raw.MessagesGetFullChat(ctx, chatID) + if err != nil { + log.Debugf("Failed to fetch chat %d: %v", chatID, err) + return "", "" + } + for _, item := range result.GetChats() { + if chat, ok := item.(*tg.Chat); ok && chat.ID == chatID { + return chat.Title, "" + } + } + return "", "" +} + +func fetchUserInfo(ctx *ext.Context, userID int64) (string, string) { + peer, err := ctx.ResolveInputPeerById(userID) + if err != nil { + log.Debugf("Failed to resolve user %d: %v", userID, err) + return "", "" + } + userPeer, ok := peer.(*tg.InputPeerUser) + if !ok { + return "", "" + } + users, err := ctx.Raw.UsersGetUsers(ctx, []tg.InputUserClass{ + &tg.InputUser{UserID: userID, AccessHash: userPeer.AccessHash}, + }) + if err != nil { + log.Debugf("Failed to fetch user %d: %v", userID, err) + return "", "" + } + for _, item := range users { + if user, ok := item.(*tg.User); ok && user.ID == userID { + return displayName(user), user.Username + } + } + return "", "" +} diff --git a/common/utils/tgutil/context.go b/common/utils/tgutil/context.go index a0ce1f41..27759794 100644 --- a/common/utils/tgutil/context.go +++ b/common/utils/tgutil/context.go @@ -3,7 +3,7 @@ package tgutil import ( "context" - "github.com/celestix/gotgproto/ext" + "github.com/krau/mygotg/ext" ) type contextKey struct{} diff --git a/common/utils/tgutil/media.go b/common/utils/tgutil/media.go index 33fb5ea4..1efb8fe7 100644 --- a/common/utils/tgutil/media.go +++ b/common/utils/tgutil/media.go @@ -2,11 +2,29 @@ package tgutil import ( "fmt" + "strings" "github.com/gabriel-vasile/mimetype" "github.com/gotd/td/tg" ) +func GetOriginalFileName(media tg.MessageMediaClass) string { + v, ok := media.(*tg.MessageMediaDocument) + if !ok { + return "" + } + f, ok := v.Document.AsNotEmpty() + if !ok { + return "" + } + for _, attribute := range f.Attributes { + if name, ok := attribute.(*tg.DocumentAttributeFilename); ok { + return strings.TrimSpace(name.GetFileName()) + } + } + return "" +} + func GetMediaFileName(media tg.MessageMediaClass) (string, error) { switch v := media.(type) { case *tg.MessageMediaPhoto: diff --git a/common/utils/tgutil/message.go b/common/utils/tgutil/message.go index 8890e2e7..1832dcae 100644 --- a/common/utils/tgutil/message.go +++ b/common/utils/tgutil/message.go @@ -8,7 +8,6 @@ import ( "unicode" "unicode/utf16" - "github.com/celestix/gotgproto/ext" "github.com/duke-git/lancet/v2/maputil" "github.com/duke-git/lancet/v2/mathutil" "github.com/duke-git/lancet/v2/slice" @@ -16,11 +15,13 @@ import ( "github.com/duke-git/lancet/v2/validator" "github.com/gabriel-vasile/mimetype" "github.com/gotd/td/constant" + "github.com/gotd/td/telegram/message/markup" "github.com/gotd/td/tg" "github.com/krau/SaveAny-Bot/common/cache" "github.com/krau/SaveAny-Bot/common/i18n" "github.com/krau/SaveAny-Bot/common/i18n/i18nk" "github.com/krau/SaveAny-Bot/common/utils/strutil" + "github.com/krau/mygotg/ext" "github.com/rs/xid" ) @@ -95,10 +96,18 @@ func GenFileNameFromMessage(message tg.Message) string { return filename + ext } -func BuildCancelButton(taskID string) tg.KeyboardButtonClass { - return &tg.KeyboardButtonCallback{ - Text: i18n.T(i18nk.BotMsgCommonCancelButtonText, nil), - Data: fmt.Appendf(nil, "cancel %s", taskID), +func BuildCancelButton(taskID string) tg.KeyboardInlineButton { + return markup.Callback( + i18n.T(i18nk.BotMsgCommonCancelButtonText, nil), + fmt.Appendf(nil, "cancel %s", taskID), + ) +} + +func BuildCancelMarkup(taskID string) *tg.ReplyInlineMarkup { + return &tg.ReplyInlineMarkup{ + Rows: []tg.KeyboardInlineButtonRow{{ + Buttons: []tg.KeyboardInlineButton{BuildCancelButton(taskID)}, + }}, } } @@ -192,10 +201,12 @@ func getMessagesRange(ctx *ext.Context, chatID int64, minId, maxId int) ([]*tg.M return result, nil } -func getMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) { +func getMessageByID(ctx *ext.Context, chatID int64, msgID int, useCache bool) (*tg.Message, error) { key := fmt.Sprintf("tgmsg:%d:%d:%d", ctx.Self.ID, chatID, msgID) - if msg, ok := cache.Get[*tg.Message](key); ok { - return msg, nil + if useCache { + if msg, ok := cache.Get[*tg.Message](key); ok { + return msg, nil + } } msgs, err := ctx.GetMessages(chatID, []tg.InputMessageClass{ &tg.InputMessageID{ID: msgID}, @@ -216,25 +227,57 @@ func getMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, err } func GetMessageByID(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) { - // we don't know what the input chatID is bot api style(e.g. channel with -100 prefix) or plain tdlib style(no any prefix and every id is positive) - if msg, err := getMessageByID(ctx, chatID, msgID); err == nil { + return lookupMessageByID(ctx, chatID, msgID, true) +} + +// GetMessageByIDFresh fetches the message from Telegram instead of the cache. +// Use it when the caller needs current file references, for example when +// retrying a failed download. +func GetMessageByIDFresh(ctx *ext.Context, chatID int64, msgID int) (*tg.Message, error) { + return lookupMessageByID(ctx, chatID, msgID, false) +} + +// GetMessageByIDFreshAny tries each client context in order and returns the +// message with the client that fetched it. It bypasses the message cache so +// file references are current. +func GetMessageByIDFreshAny(ctxs []*ext.Context, chatID int64, msgID int) (*tg.Message, *ext.Context, error) { + var lastErr error + for _, ctx := range ctxs { + if ctx == nil { + continue + } + msg, err := GetMessageByIDFresh(ctx, chatID, msgID) + if err == nil { + return msg, ctx, nil + } + lastErr = err + } + if lastErr == nil { + lastErr = fmt.Errorf("no telegram client available") + } + return nil, nil, lastErr +} + +func lookupMessageByID(ctx *ext.Context, chatID int64, msgID int, useCache bool) (*tg.Message, error) { + // we don't know if the input chatID is bot api style(e.g. channel with -100 prefix) or plain tdlib style(no any prefix and every id is positive) + if msg, err := getMessageByID(ctx, chatID, msgID, useCache); err == nil { return msg, nil } in := constant.TDLibPeerID(chatID) plain := in.ToPlain() var channel constant.TDLibPeerID channel.Channel(plain) - if msg, err := getMessageByID(ctx, int64(channel), msgID); err == nil { + if msg, err := getMessageByID(ctx, int64(channel), msgID, useCache); err == nil { return msg, nil } var chat constant.TDLibPeerID chat.Chat(plain) - if msg, err := getMessageByID(ctx, int64(chat), msgID); err == nil { + if msg, err := getMessageByID(ctx, int64(chat), msgID, useCache); err == nil { return msg, nil } var userID constant.TDLibPeerID userID.User(plain) - if msg, err := getMessageByID(ctx, int64(userID), msgID); err == nil { + if msg, err := getMessageByID(ctx, int64(userID), msgID, useCache); err == nil { return msg, nil } diff --git a/common/utils/tgutil/peer.go b/common/utils/tgutil/peer.go index ddf0ea82..f8e02ce1 100644 --- a/common/utils/tgutil/peer.go +++ b/common/utils/tgutil/peer.go @@ -1,15 +1,47 @@ package tgutil -import "github.com/gotd/td/tg" +import ( + "github.com/gotd/td/constant" + "github.com/gotd/td/tg" + tgstorage "github.com/krau/mygotg/storage" +) +// MarkedChannelID converts a plain channel ID to the Bot API style +// -100 form used across the database and HTTP API. +func MarkedChannelID(id int64) int64 { + var peerID constant.TDLibPeerID + peerID.Channel(id) + return int64(peerID) +} + +// MarkedChatID converts a plain basic-group ID to its negative Bot API form. +func MarkedChatID(id int64) int64 { + var peerID constant.TDLibPeerID + peerID.Chat(id) + return int64(peerID) +} + +// MarkedPeerID converts a plain peer ID to the Bot API style form for its type. +func MarkedPeerID(id int64, typ int) int64 { + switch tgstorage.EntityType(typ) { + case tgstorage.TypeChannel: + return MarkedChannelID(id) + case tgstorage.TypeChat: + return MarkedChatID(id) + default: + return id + } +} + +// ChatIdFromPeer returns the Bot API style ID of a message or callback peer. func ChatIdFromPeer(peer tg.PeerClass) int64 { - switch peer := peer.(type) { + switch p := peer.(type) { case *tg.PeerChannel: - return peer.ChannelID - case *tg.PeerUser: - return peer.UserID + return MarkedChannelID(p.ChannelID) case *tg.PeerChat: - return peer.ChatID + return MarkedChatID(p.ChatID) + case *tg.PeerUser: + return p.UserID default: return 0 } diff --git a/common/utils/tgutil/progress_edit.go b/common/utils/tgutil/progress_edit.go index 5648b7b0..a9352bfc 100644 --- a/common/utils/tgutil/progress_edit.go +++ b/common/utils/tgutil/progress_edit.go @@ -16,16 +16,38 @@ type ProgressSender func(ctx context.Context, chatID int64, req *tg.MessagesEdit const progressEditInterval = time.Second -var progressEditLimiter = rate.NewLimiter(rate.Every(progressEditInterval), 1) +var progressEditLimiters = struct { + sync.Mutex + byChat map[int64]*rate.Limiter +}{byChat: make(map[int64]*rate.Limiter)} + +func progressEditLimiter(chatID int64) *rate.Limiter { + progressEditLimiters.Lock() + defer progressEditLimiters.Unlock() + limiter := progressEditLimiters.byChat[chatID] + if limiter == nil { + limiter = rate.NewLimiter(rate.Every(progressEditInterval), 1) + progressEditLimiters.byChat[chatID] = limiter + } + return limiter +} func SendProgressEdit(ctx context.Context, chatID int64, req *tg.MessagesEditMessageRequest) error { - if err := progressEditLimiter.Wait(context.Background()); err != nil { + if err := progressEditLimiter(chatID).Wait(ctx); err != nil { return err } ext := ExtFromContext(ctx) if ext == nil { return nil } + if ext.Context != nil && ext.Context.Err() != nil { + // The final progress edit can run after the task context is canceled. + // Keep the Telegram client and other context values, but detach the + // cancellation signal so EditMessage can still be sent. + clone := *ext + clone.Context = context.WithoutCancel(ext.Context) + ext = &clone + } if _, err := ext.EditMessage(chatID, req); err != nil { return err } @@ -53,6 +75,12 @@ func NewProgressEditQueue(sender ProgressSender) *ProgressEditQueue { } func (q *ProgressEditQueue) Submit(ctx context.Context, chatID int64, req *tg.MessagesEditMessageRequest, final bool) { + if final { + // The final edit may run after the task context is canceled by the + // queue. Keep its values (including the Telegram client) but not the + // cancellation signal, otherwise the message stays in an old phase. + ctx = context.WithoutCancel(ctx) + } job := progressEditJob{ctx: ctx, chatID: chatID, req: req, final: final} q.start.Do(func() { go q.run() }) for { diff --git a/common/utils/tgutil/resolve.go b/common/utils/tgutil/resolve.go index b5e975c0..00b74de9 100644 --- a/common/utils/tgutil/resolve.go +++ b/common/utils/tgutil/resolve.go @@ -6,24 +6,42 @@ import ( "strconv" "strings" - "github.com/celestix/gotgproto/ext" "github.com/duke-git/lancet/v2/validator" + "github.com/gotd/td/constant" "github.com/gotd/td/tg" + "github.com/krau/mygotg/ext" ) func ParseChatID(ctx *ext.Context, idOrUsername string) (int64, error) { - idOrUsername = strings.TrimPrefix(idOrUsername, "@") + idOrUsername = strings.TrimPrefix(strings.TrimSpace(idOrUsername), "@") + if idOrUsername == "" { + return 0, fmt.Errorf("chat ID is empty") + } if validator.IsIntStr(idOrUsername) { - chatID, err := strconv.Atoi(idOrUsername) + chatID, err := strconv.ParseInt(idOrUsername, 10, 64) if err != nil { return 0, err } - return int64(chatID), nil + if chatID == 0 { + return 0, fmt.Errorf("chat ID must not be zero") + } + if chatID < 0 { + peerID := constant.TDLibPeerID(chatID) + if !peerID.IsChannel() && !peerID.IsChat() { + return 0, fmt.Errorf("invalid negative chat ID: %d", chatID) + } + return chatID, nil + } + // A plain ID is a user ID unless the peer cache already knows that + // this plain ID belongs to a channel or basic group. + if peer := ctx.PeerStorage.GetPeerById(chatID); peer != nil && peer.ID != 0 { + return MarkedPeerID(peer.ID, peer.Type), nil + } + return chatID, nil } username := idOrUsername - peer := ctx.PeerStorage.GetPeerByUsername(username) - if peer != nil && peer.ID != 0 { - return peer.ID, nil + if peer := ctx.PeerStorage.GetPeerByUsername(username); peer != nil && peer.ID != 0 { + return MarkedPeerID(peer.ID, peer.Type), nil } chat, err := ctx.ResolveUsername(username) if err != nil { @@ -36,7 +54,29 @@ func ParseChatID(ctx *ext.Context, idOrUsername string) (int64, error) { if chatID == 0 { return 0, fmt.Errorf("chat ID is zero for username: %s", idOrUsername) } - return chatID, nil + switch { + case chat.IsAChannel(): + return MarkedChannelID(chatID), nil + case chat.IsAChat(): + return MarkedChatID(chatID), nil + default: + return chatID, nil + } +} + +// ParseChannelID parses a t.me/c/ channel reference as a Bot API style ID. +func ParseChannelID(_ *ext.Context, value string) (int64, error) { + value = strings.TrimSpace(value) + if id, err := strconv.ParseInt(value, 10, 64); err == nil { + if id == 0 { + return 0, fmt.Errorf("channel ID must not be zero") + } + if id < 0 { + return id, nil + } + return MarkedChannelID(id), nil + } + return 0, fmt.Errorf("invalid channel ID: %s", value) } // return: ChatID, MessageID, error @@ -91,7 +131,13 @@ func ParseMessageLink(ctx *ext.Context, link string) (int64, int, error) { if paths[0] != "c" { chatPart = paths[0] } - chatID, err := ParseChatID(ctx, chatPart) + var chatID int64 + var err error + if paths[0] == "c" { + chatID, err = ParseChannelID(ctx, chatPart) + } else { + chatID, err = ParseChatID(ctx, chatPart) + } if err != nil { return 0, 0, fmt.Errorf("failed to parse chat ID: %w", err) } @@ -105,7 +151,7 @@ func ParseMessageLink(ctx *ext.Context, link string) (int64, int, error) { if paths[0] != "c" { return 0, 0, fmt.Errorf("invalid message link format: %s", link) } - chatID, err := ParseChatID(ctx, paths[1]) + chatID, err := ParseChannelID(ctx, paths[1]) if err != nil { return 0, 0, fmt.Errorf("failed to parse chat ID: %w", err) } diff --git a/common/utils/tgutil/sender.go b/common/utils/tgutil/sender.go new file mode 100644 index 00000000..2239914a --- /dev/null +++ b/common/utils/tgutil/sender.go @@ -0,0 +1,56 @@ +package tgutil + +import ( + "github.com/gotd/td/tg" + "github.com/krau/mygotg/ext" +) + +// SenderID returns the user responsible for the update. Callback queries carry +// it natively; message updates resolve it from the sender peer. +func SenderID(u *ext.Update) int64 { + if u == nil { + return 0 + } + if u.CallbackQuery != nil { + return u.CallbackQuery.GetUserID() + } + message := u.EffectiveMessage + // Outgoing updates in private chats belong to the chat peer, not to the + // bot that sent them. + if message != nil && message.Out { + return u.GetUserChat().GetID() + } + if user := u.EffectiveUser(); user != nil { + return user.ID + } + if message != nil { + if from, ok := message.GetFromID(); ok { + if peer, ok := from.(*tg.PeerUser); ok { + return peer.UserID + } + } + } + return u.GetUserChat().GetID() +} + +// ChatID returns the chat that replies and edits must target. In private chats +// this equals the sender ID; in groups and supergroups it is the group chat. +// Peer IDs are used directly so callback updates without entity maps still +// resolve their target chat. +func ChatID(u *ext.Update) int64 { + if u == nil { + return 0 + } + if message := u.EffectiveMessage; message != nil { + if id := ChatIdFromPeer(message.PeerID); id != 0 { + return id + } + } + if cb := u.CallbackQuery; cb != nil { + if id := ChatIdFromPeer(cb.Peer); id != 0 { + return id + } + return cb.GetUserID() + } + return u.EffectiveChat().GetID() +} diff --git a/common/utils/tphutil/tph.go b/common/utils/tphutil/tph.go index d2647661..525bc1ae 100644 --- a/common/utils/tphutil/tph.go +++ b/common/utils/tphutil/tph.go @@ -1,41 +1,53 @@ package tphutil import ( + "context" "encoding/json" + "errors" + "fmt" + "net/url" "strings" "sync" + "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/pkg/telegraph" ) -var ( - tphClient *telegraph.Client - once sync.Once -) +var defaultClient = struct { + sync.Mutex + key string + client *telegraph.Client +}{} func DefaultClient() *telegraph.Client { - once.Do(func() { - tphClient = initDefault() - }) - return tphClient -} - -func initDefault() *telegraph.Client { - var client *telegraph.Client - if config.C().Telegram.Proxy.Enable && config.C().Telegram.Proxy.URL != "" { - proxyUrl := config.C().Telegram.Proxy.URL - var err error - client, err = telegraph.NewClientWithProxy(proxyUrl) - if err != nil { - client = telegraph.NewClient() - } - } else { + cfg := config.C().Telegram.Proxy + key := "direct" + if cfg.Enable && cfg.URL != "" { + key = cfg.URL + } + defaultClient.Lock() + defer defaultClient.Unlock() + if defaultClient.client != nil && defaultClient.key == key { + return defaultClient.client + } + client, err := newClient(key) + if err != nil { + log.Warn("Failed to create Telegraph client with proxy; using direct client", "error", err) client = telegraph.NewClient() } + defaultClient.client = client + defaultClient.key = key return client } +func newClient(key string) (*telegraph.Client, error) { + if key == "direct" { + return telegraph.NewClient(), nil + } + return telegraph.NewClientWithProxy(key) +} + func GetNodeImages(node telegraph.Node) []string { var srcs []string @@ -62,3 +74,55 @@ func GetNodeImages(node telegraph.Node) []string { } return srcs } + +// ParsePagePath validates a Telegraph page URL and returns its path. Bot +// messages require HTTPS; the HTTP API also accepts legacy HTTP links. +func ParsePagePath(pageURL string) (string, error) { + return parsePagePath(pageURL, false) +} + +func parsePagePath(pageURL string, allowHTTP bool) (string, error) { + u, err := url.Parse(pageURL) + if err != nil { + return "", fmt.Errorf("invalid telegraph URL: %w", err) + } + if u.Scheme != "https" && !(allowHTTP && u.Scheme == "http") { + return "", fmt.Errorf("invalid telegraph URL scheme: %s", u.Scheme) + } + if !strings.EqualFold(u.Hostname(), "telegra.ph") && !strings.EqualFold(u.Hostname(), "telegraph.co") { + return "", fmt.Errorf("invalid telegraph URL host: %s", u.Host) + } + pagePath := strings.Trim(u.EscapedPath(), "/") + if pagePath == "" || strings.Contains(pagePath, "/") { + return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path) + } + pagePath, err = url.PathUnescape(pagePath) + if err != nil { + return "", fmt.Errorf("failed to unescape telegraph path: %w", err) + } + pagePath = strings.TrimSpace(pagePath) + if pagePath == "" || strings.Contains(pagePath, "/") { + return "", fmt.Errorf("invalid telegraph URL path: %s", u.Path) + } + return pagePath, nil +} + +// ExtractImages fetches a Telegraph page and returns its image URLs and path. +func ExtractImages(ctx context.Context, pageURL string) ([]string, string, error) { + pagePath, err := parsePagePath(pageURL, true) + if err != nil { + return nil, "", err + } + page, err := DefaultClient().GetPage(ctx, pagePath) + if err != nil { + return nil, "", fmt.Errorf("failed to get telegraph page: %w", err) + } + images := make([]string, 0) + for _, elem := range page.Content { + images = append(images, GetNodeImages(elem)...) + } + if len(images) == 0 { + return nil, "", errors.New("no images found in telegraph page") + } + return images, pagePath, nil +} diff --git a/config.example.toml b/config.example.toml deleted file mode 100644 index 9c07f3ab..00000000 --- a/config.example.toml +++ /dev/null @@ -1,135 +0,0 @@ -# 创建文件时,若需要保留中文注释,请务必确保本文件编码为 UTF-8 ,否则会无法读取。 -# When creating this file, ensure the encoding is UTF-8 to preserve Chinese comments, otherwise it may fail to read. -# 更详细的配置请在 https://sabot.unv.app/deployment/configuration 查看 -# For more detailed configuration, see https://sabot.unv.app/deployment/configuration -workers = 4 # 同时下载文件数 / Number of concurrent downloads -retry = 3 # 下载失败重试次数 / Number of retries on download failure -threads = 4 # 单个任务下载使用的最大线程数 / Max threads per download task -stream = false # 使用流式传输模式, 建议仅在硬盘空间十分有限时使用. / Stream mode, recommended only when disk space is very limited. - -[log] -# 日志级别, 可选: debug, info, warn, error, fatal / Log level, options: debug, info, warn, error, fatal -level = "debug" - -[telegram] -# Bot Token -# 更换 Bot Token 后请删除会话数据库文件 (默认路径为 data/session.db ) -# After changing Bot Token, delete the session database file (default path: data/session.db) -token = "" -# Telegram API 配置, 若不配置也可运行, 将使用默认的 API ID 和 API HASH -# Telegram API config, optional - defaults will be used if not set -# 推荐使用自己的 API ID 和 API HASH (https://my.telegram.org) -# Recommended: use your own API ID and API HASH (https://my.telegram.org) -# app_id = 1025907 -# app_hash = "452b0359b988148995f22ff0f4229750" -[telegram.proxy] -# 启用代理连接 telegram / Enable proxy for Telegram connection -enable = false -url = "socks5://127.0.0.1:7890" - -# Aria2 配置 / Aria2 configuration -[aria2] -# 启用 Aria2 下载支持 / Enable Aria2 download support -enable = false -# Aria2 RPC URL -url = "http://localhost:6800/jsonrpc" -# Aria2 RPC Secret (如果配置了 rpc-secret) / Aria2 RPC Secret (if rpc-secret is configured) -secret = "" -# 转存完成后删除 Aria2 下载的本地文件 / Delete local files downloaded by Aria2 after transfer -remove_after_transfer = true - -# yt-dlp 视频下载配置 / yt-dlp video download configuration -[ytdlp] -# 默认下载的最高视频清晰度 (按高度限制), 如 1080, 720, 480; 0 表示不限制 (下载最佳画质) -# Max video resolution by height, e.g. 1080, 720, 480; 0 means no limit (best available) -# 仅在使用 /ytdlp 命令且未手动指定任何参数时生效 -# Only applies when using the /ytdlp command without manually specified flags -max_height = 1080 -# 直接指定 yt-dlp format 选择表达式, 留空则使用 max_height -# Raw yt-dlp format selector; leave empty to use max_height -# 设置后优先级高于 max_height, 例如: "bv*[height<=720]+ba/b" -# Takes precedence over max_height when set, e.g. "bv*[height<=720]+ba/b" -format = "" -# 下载后转封装的视频容器格式, 留空则不转封装. 默认 mp4 -# Target video container to recode into after download; empty disables recoding. Default mp4 -recode = "mp4" -# 下载文件名模板 (yt-dlp output template), 默认使用视频标题 -# 目录始终由 bot 管理, 模板中的路径相对于 bot 的临时下载目录 -# Filename template for downloads (yt-dlp output template), defaults to the video title -# The download directory is always managed by the bot; paths inside the template are relative to its temp directory -# 例如 / e.g. "%(uploader)s - %(title)s.%(ext)s" -filename_template = "%(title)s.%(ext)s" -# 将文件名限制为 ASCII 字符 (yt-dlp --restrict-filenames), 会丢弃标题中的非拉丁字符、空格和 & -# Restrict file names to ASCII characters (yt-dlp --restrict-filenames): drops non-latin characters, spaces and & from titles -restrict_filenames = false - -# 解析器配置 / Parser configuration -[parser] -# 启用 JS 解析器插件 (Go 内置解析器默认启用) -# Enable JS parser plugins (built-in Go parsers are enabled by default) -plugin_enable = false -# 插件目录, 可以是多个目录 / Plugin directories, multiple allowed -plugin_dirs = ["./plugins"] -# 解析器默认代理 / Default proxy for parsers -proxy = "" - -# Twitter/X 解析器配置 / Twitter/X parser configuration -[parser.twitter] -# 自定义 API 域名 / Custom API domain -api_domain = "api.fxtwitter.com" -# 单独为此解析器指定代理 (留空则使用 [parser] 中的 proxy) -# Proxy for this parser only (leave empty to use the proxy from [parser]) -# proxy = "http://127.0.0.1:7890" - -# Kemono 解析器配置 (暂无可配置项, 留空即可) -# Kemono parser configuration (no options yet, leave empty) -[parser.kemono] - -# HTTP API 配置 / HTTP API configuration -[api] -# 启用 HTTP API / Enable HTTP API -enable = false -# 监听地址 / Listen address -host = "0.0.0.0" -# 监听端口 / Listen port -port = 8080 -# 认证 Token (必需) / Auth token (required) -token = "" - -# 存储列表 / Storage list -[[storages]] -# 标识名, 需要唯一 / Identifier name, must be unique -name = "本机1" -# 存储类型, 目前可用: local, alist, webdav, s3, telegram -# Storage type, available: local, alist, webdav, s3, telegram -type = "local" -# 启用存储 / Enable storage -enable = true -# 文件保存根路径 / Root path for saving files -base_path = "./downloads" - -[[storages]] -name = "MyWebdav" -type = "webdav" -enable = false -base_path = '/path/telegram' -url = 'https://example.com/dav' -username = 'username' -password = 'password' - -# 用户列表 / User list -[[users]] -# telegram user id -id = 114514 -# 存储过滤列表, 元素为存储标识名. / Storage filter list, elements are storage identifier names. -# 将该列表留空并开启黑名单过滤模式以允许使用所有存储,此处示例为黑名单模式,用户 114514 可使用所有存储 -# Leave empty with blacklist mode enabled to allow all storages; this example uses blacklist mode, user 114514 can use all storages -storages = [] -# 使用列表过滤黑名单模式,反之则为白名单,白名单请在列表中指定可用的存储. -# Blacklist filter mode; set false for whitelist mode and specify allowed storages in the list. -blacklist = true - -[[users]] -id = 123456 -storages = ["本机1"] -blacklist = false # 使用白名单模式,此时,用户 123456 仅可使用标识名为 '本地1' 的存储 / Whitelist mode: user 123456 can only use the storage named '本机1' diff --git a/config/aria2.go b/config/aria2.go new file mode 100644 index 00000000..fc1b96cb --- /dev/null +++ b/config/aria2.go @@ -0,0 +1,8 @@ +package config + +type aria2Config struct { + Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` + Url string `toml:"url" mapstructure:"url" json:"url"` + Secret string `toml:"secret" mapstructure:"secret" json:"secret"` + KeepFile bool `toml:"keep_file" mapstructure:"keep_file" json:"keep_file"` +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..8c8367b0 --- /dev/null +++ b/config/config.go @@ -0,0 +1,286 @@ +package config + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "net" + "net/http" + "net/url" + "path/filepath" + "sync/atomic" + "time" + + "github.com/krau/SaveAny-Bot/config/storage" + "golang.org/x/net/proxy" +) + +type Config struct { + Lang string `toml:"lang" mapstructure:"lang" json:"lang"` + Workers int `toml:"workers" mapstructure:"workers" json:"workers"` + Retry int `toml:"retry" mapstructure:"retry" json:"retry"` + NoCleanCache bool `toml:"no_clean_cache" mapstructure:"no_clean_cache" json:"no_clean_cache"` + Threads int `toml:"threads" mapstructure:"threads" json:"threads"` + // DownloadThreads overrides automatic per-file chunk concurrency when > 0. + DownloadThreads int `toml:"download_threads" mapstructure:"download_threads" json:"download_threads"` + // MaxDownloadThreads caps automatic per-file chunk concurrency. + MaxDownloadThreads int `toml:"max_download_threads" mapstructure:"max_download_threads" json:"max_download_threads"` + // TransferBudget caps concurrent Telegram part RPCs; 0 derives workers * threads. + TransferBudget int `toml:"transfer_budget" mapstructure:"transfer_budget" json:"transfer_budget"` + // RelayMinSizeMB controls the smallest file eligible for helper downloads. + RelayMinSizeMB int `toml:"relay_min_size_mb" mapstructure:"relay_min_size_mb" json:"relay_min_size_mb"` + // ChunkRetries is the retry count for transient errors on one file chunk. + ChunkRetries int `toml:"chunk_retries" mapstructure:"chunk_retries" json:"chunk_retries"` + Stream bool `toml:"stream" mapstructure:"stream" json:"stream"` + Proxy string `toml:"proxy" mapstructure:"proxy" json:"proxy"` + Log logConfig `toml:"log" mapstructure:"log" json:"log"` + Aria2 aria2Config `toml:"aria2" mapstructure:"aria2" json:"aria2"` + Cache cacheConfig `toml:"cache" mapstructure:"cache" json:"cache"` + Users []UserConfig `toml:"users" mapstructure:"users" json:"-"` + Temp tempConfig `toml:"temp" mapstructure:"temp" json:"temp"` + DB dbConfig `toml:"db" mapstructure:"db" json:"db"` + Telegram telegramConfig `toml:"telegram" mapstructure:"telegram" json:"telegram"` + Storages []storage.StorageConfig `toml:"-" mapstructure:"-" json:"-"` + Parser parserConfig `toml:"parser" mapstructure:"parser" json:"parser"` + Hook hookConfig `toml:"hook" mapstructure:"hook" json:"hook"` + Ytdlp YtdlpConfig `toml:"ytdlp" mapstructure:"ytdlp" json:"ytdlp"` + Web webConfig `toml:"web" mapstructure:"web" json:"web"` + Metadata metadataConfig `toml:"metadata" mapstructure:"metadata" json:"metadata"` + Media mediaConfig `toml:"media" mapstructure:"media" json:"media"` +} + +type metadataConfig struct { + Enabled bool `toml:"enabled" mapstructure:"enabled" json:"enabled"` + IncludeText bool `toml:"include_text" mapstructure:"include_text" json:"include_text"` + Sidecar bool `toml:"sidecar" mapstructure:"sidecar" json:"sidecar"` +} + +type mediaConfig struct { + // AllowedMediaTypes is empty when all supported media types are allowed. + AllowedMediaTypes []string `toml:"allowed_media_types" mapstructure:"allowed_media_types" json:"allowed_media_types"` +} + +type webConfig struct { + Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` + Listen string `toml:"listen" mapstructure:"listen" json:"listen"` + APIToken string `toml:"api_token" mapstructure:"api_token" json:"api_token"` + SecureCookie bool `toml:"secure_cookie" mapstructure:"secure_cookie" json:"secure_cookie"` +} + +const ( + defaultBotSessionFile = "bot_session.db" + defaultUserbotSessionFile = "userbot_session.db" + defaultWebListen = "127.0.0.1:28080" +) + +var current atomic.Pointer[Config] + +// DefaultBotSessionPath returns the v1 bot session database path. +func DefaultBotSessionPath(dataDir string) string { + return filepath.Join(dataDir, defaultBotSessionFile) +} + +// DefaultUserbotSessionPath returns the v1 userbot session database path. +func DefaultUserbotSessionPath(dataDir string) string { + return filepath.Join(dataDir, defaultUserbotSessionFile) +} + +func init() { + cfg := Default("./data") + current.Store(&cfg) +} + +// C returns a snapshot of the current configuration. Callers must not mutate it. +func C() Config { + return *current.Load() +} + +// TransferBudget is the upper bound for concurrent Telegram file-part RPCs +// (download and upload combined). It keeps the per-file thread setting from +// multiplying with task and element concurrency. +func TransferBudget() int { + cfg := current.Load() + if cfg.TransferBudget > 0 { + return cfg.TransferBudget + } + return max(cfg.Workers, 1) * max(cfg.Threads, 1) +} + +// Set replaces the current configuration snapshot. +func Set(cfg Config) { + current.Store(&cfg) +} + +func DefaultsMap(dataDir string) map[string]any { + return map[string]any{ + "lang": "zh-Hans", + "workers": 3, + "retry": 3, + "no_clean_cache": false, + "threads": 4, + "download_threads": 0, + "max_download_threads": 16, + "transfer_budget": 0, + "relay_min_size_mb": 8, + "chunk_retries": 20, + "log.level": "debug", + + "cache.ttl": 86400, + "cache.num_counters": 1e5, + "cache.max_cost": 1e6, + + "telegram.app_id": 1025907, + "telegram.app_hash": "452b0359b988148995f22ff0f4229750", + "telegram.rpc_retry": 5, + "telegram.userbot.enable": false, + "telegram.userbot.session": DefaultUserbotSessionPath(dataDir), + "telegram.relay_chat_id": 0, + "telegram.relay_keep_messages": true, + + "temp.base_path": "cache/", + + "db.session": DefaultBotSessionPath(dataDir), + + "web.enable": true, + "web.listen": defaultWebListen, + "web.api_token": "", + "web.secure_cookie": false, + + "ytdlp.recode": "mp4", + "ytdlp.filename_template": DefaultYtdlpFilenameTemplate, + "ytdlp.max_height": 0, + "ytdlp.format": "", + "ytdlp.restrict_filenames": false, + + "metadata.enabled": false, + "metadata.include_text": true, + "metadata.sidecar": false, + "media.allowed_media_types": []string{}, + } +} + +func Default(dataDir string) Config { + return Config{ + Lang: "zh-Hans", + Workers: 3, + Retry: 3, + Threads: 4, + DownloadThreads: 0, + MaxDownloadThreads: 16, + TransferBudget: 0, + RelayMinSizeMB: 8, + ChunkRetries: 20, + Log: logConfig{Level: "debug"}, + Cache: cacheConfig{ + TTL: 86400, + NumCounters: 1e5, + MaxCost: 1e6, + }, + Temp: tempConfig{BasePath: "cache/"}, + DB: dbConfig{Session: DefaultBotSessionPath(dataDir)}, + Telegram: telegramConfig{ + AppID: 1025907, + AppHash: "452b0359b988148995f22ff0f4229750", + RpcRetry: 5, + Userbot: userbotConfig{Session: DefaultUserbotSessionPath(dataDir)}, + RelayKeepMessages: true, + }, + Ytdlp: YtdlpConfig{ + Recode: "mp4", + FilenameTemplate: DefaultYtdlpFilenameTemplate, + }, + Web: webConfig{ + Enable: true, + Listen: defaultWebListen, + }, + Metadata: metadataConfig{IncludeText: true}, + Media: mediaConfig{}, + } +} + +func (c Config) GetStorageByName(name string) storage.StorageConfig { + for _, st := range c.Storages { + if st.GetName() == name { + return st + } + } + return nil +} + +func (c Config) GetStorageIDByName(name string) uint { + if st := c.GetStorageByName(name); st != nil { + return st.GetID() + } + return 0 +} + +func (c Config) GetStorageByID(id uint) storage.StorageConfig { + if id == 0 { + return nil + } + for _, st := range c.Storages { + if st.GetID() == id { + return st + } + } + return nil +} + +func (c Config) GetStorageNameByID(id uint) string { + if id == 0 { + return "" + } + if st := c.GetStorageByID(id); st != nil { + return st.GetName() + } + return "" +} + +// ProxyTransport builds the HTTP transport used for the configured proxy. +func ProxyTransport(proxyStr string) (*http.Transport, error) { + proxyURL, err := url.Parse(proxyStr) + if err != nil { + return nil, err + } + transport := &http.Transport{ + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + switch proxyURL.Scheme { + case "http", "https": + transport.Proxy = http.ProxyURL(proxyURL) + case "socks5", "socks5h": + dialer, err := proxy.FromURL(proxyURL, proxy.Direct) + if err != nil { + return nil, err + } + transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + return dialer.(proxy.ContextDialer).DialContext(ctx, network, addr) + } + default: + return nil, fmt.Errorf("unsupported proxy type: %s", proxyURL.Scheme) + } + return transport, nil +} + +// GeneratePassword returns a URL-safe random password with ~120 bits of entropy. +func GeneratePassword() string { + return randomBase64(15) +} + +// GenerateToken returns a URL-safe token with 256 bits of entropy. +func GenerateToken() string { + return randomBase64(32) +} + +func randomBase64(size int) string { + buf := make([]byte, size) + if _, err := rand.Read(buf); err != nil { + panic(fmt.Sprintf("failed to read random bytes: %v", err)) + } + return base64.RawURLEncoding.EncodeToString(buf) +} diff --git a/config/db.go b/config/db.go index ba7b2a60..2b80686b 100644 --- a/config/db.go +++ b/config/db.go @@ -1,6 +1,5 @@ package config type dbConfig struct { - Path string `toml:"path" mapstructure:"path"` - Session string `toml:"session" mapstructure:"session"` + Session string `toml:"session" mapstructure:"session" json:"session"` } diff --git a/config/flags.go b/config/flags.go deleted file mode 100644 index e0bc0b3f..00000000 --- a/config/flags.go +++ /dev/null @@ -1,75 +0,0 @@ -package config - -import ( - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func RegisterFlags(cmd *cobra.Command) { - flags := cmd.Flags() - - flags.StringP("config", "c", "", "config file path") - flags.StringP("lang", "l", "", "language (e.g., zh-Hans, en)") - flags.IntP("workers", "w", 0, "number of workers") - flags.Int("retry", 0, "retry times") - flags.Int("threads", 0, "number of threads") - flags.Bool("stream", false, "enable stream mode") - flags.Bool("no-clean-cache", false, "do not clean cache on exit") - flags.String("proxy", "", "proxy URL (http, https, socks5, socks5h)") - flags.String("log-level", "", "log level (trace/debug, info, warn, error, fatal)") - - flags.String("telegram-token", "", "telegram bot token") - flags.Int("telegram-app-id", 0, "telegram app id") - flags.String("telegram-app-hash", "", "telegram app hash") - flags.Int("telegram-rpc-retry", 0, "telegram rpc retry times") - flags.Bool("telegram-userbot-enable", false, "enable userbot") - flags.String("telegram-userbot-session", "", "userbot session path") - flags.Bool("telegram-proxy-enable", false, "enable telegram proxy") - flags.String("telegram-proxy-url", "", "telegram proxy URL") - - flags.String("db-path", "", "database path") - flags.String("db-session", "", "session database path") - - flags.String("temp-base-path", "", "temp directory base path") - - flags.Bool("parser-plugin-enable", false, "enable parser plugins") - flags.StringSlice("parser-plugin-dirs", nil, "parser plugin directories") - flags.String("parser-proxy", "", "parser proxy URL") - - bindFlags(cmd) -} - -func bindFlags(cmd *cobra.Command) { - flags := cmd.Flags() - - viper.BindPFlag("lang", flags.Lookup("lang")) - viper.BindPFlag("workers", flags.Lookup("workers")) - viper.BindPFlag("retry", flags.Lookup("retry")) - viper.BindPFlag("threads", flags.Lookup("threads")) - viper.BindPFlag("stream", flags.Lookup("stream")) - viper.BindPFlag("no_clean_cache", flags.Lookup("no-clean-cache")) - viper.BindPFlag("proxy", flags.Lookup("proxy")) - viper.BindPFlag("log.level", flags.Lookup("log-level")) - - viper.BindPFlag("telegram.token", flags.Lookup("telegram-token")) - viper.BindPFlag("telegram.app_id", flags.Lookup("telegram-app-id")) - viper.BindPFlag("telegram.app_hash", flags.Lookup("telegram-app-hash")) - viper.BindPFlag("telegram.rpc_retry", flags.Lookup("telegram-rpc-retry")) - viper.BindPFlag("telegram.userbot.enable", flags.Lookup("telegram-userbot-enable")) - viper.BindPFlag("telegram.userbot.session", flags.Lookup("telegram-userbot-session")) - viper.BindPFlag("telegram.proxy.enable", flags.Lookup("telegram-proxy-enable")) - viper.BindPFlag("telegram.proxy.url", flags.Lookup("telegram-proxy-url")) - - viper.BindPFlag("db.path", flags.Lookup("db-path")) - viper.BindPFlag("db.session", flags.Lookup("db-session")) - viper.BindPFlag("temp.base_path", flags.Lookup("temp-base-path")) - - viper.BindPFlag("parser.plugin_enable", flags.Lookup("parser-plugin-enable")) - viper.BindPFlag("parser.plugin_dirs", flags.Lookup("parser-plugin-dirs")) - viper.BindPFlag("parser.proxy", flags.Lookup("parser-proxy")) -} - -func GetConfigFile(cmd *cobra.Command) string { - configFile, _ := cmd.Flags().GetString("config") - return configFile -} diff --git a/config/parser.go b/config/parser.go index b7920955..e4118b0d 100644 --- a/config/parser.go +++ b/config/parser.go @@ -4,7 +4,7 @@ type parserConfig struct { PluginEnable bool `toml:"plugin_enable" mapstructure:"plugin_enable" json:"plugin_enable"` PluginDirs []string `toml:"plugin_dirs" mapstructure:"plugin_dirs" json:"plugin_dirs"` Proxy string `toml:"proxy" mapstructure:"proxy" json:"proxy"` - ParserCfgs map[string]map[string]any `mapstructure:",remain"` + ParserCfgs map[string]map[string]any `mapstructure:",remain" json:"configs"` } func (c Config) GetParserConfigByName(name string) map[string]any { diff --git a/config/storage/factory.go b/config/storage/factory.go index 8deeab4b..920ad167 100644 --- a/config/storage/factory.go +++ b/config/storage/factory.go @@ -1,70 +1,81 @@ package storage import ( + "encoding/json" "fmt" - "reflect" storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" - "github.com/mitchellh/mapstructure" "github.com/spf13/viper" ) -var storageFactories = map[storenum.StorageType]func(cfg *BaseConfig) (StorageConfig, error){ - storenum.Local: createStorageConfig(&LocalStorageConfig{}), - storenum.Alist: createStorageConfig(&AlistStorageConfig{}), - storenum.Webdav: createStorageConfig(&WebdavStorageConfig{}), - storenum.Minio: createStorageConfig(&MinioStorageConfig{}), - storenum.S3: createStorageConfig(&S3StorageConfig{}), - storenum.Telegram: createStorageConfig(&TelegramStorageConfig{}), - storenum.Rclone: createStorageConfig(&RcloneStorageConfig{}), -} - -func createStorageConfig(configType StorageConfig) func(cfg *BaseConfig) (StorageConfig, error) { - return func(cfg *BaseConfig) (StorageConfig, error) { - configValue := reflect.New(reflect.TypeOf(configType).Elem()).Interface().(StorageConfig) +// NewStorageConfigFromJSON builds a typed storage config from JSON-encoded +// type-specific fields. Only the legacy importer still needs Viper; runtime +// storage settings come from the database as JSON. +func NewStorageConfigFromJSON(id uint, name, typ string, enable bool, raw []byte) (StorageConfig, error) { + st, err := storenum.ParseStorageType(typ) + if err != nil { + return nil, fmt.Errorf("invalid storage type %s: %w", typ, err) + } - reflect.ValueOf(configValue).Elem().FieldByName("BaseConfig").Set(reflect.ValueOf(*cfg)) + base := BaseConfig{ID: id, Name: name, Type: typ, Enable: enable} + cfg, err := newStorageConfig(st, base) + if err != nil { + return nil, err + } - if err := mapstructure.Decode(cfg.RawConfig, configValue); err != nil { - return nil, fmt.Errorf("failed to decode %s storage config: %w", cfg.Type, err) + if len(raw) > 0 { + if err := json.Unmarshal(raw, cfg); err != nil { + return nil, fmt.Errorf("failed to decode %s storage config: %w", typ, err) } + } + // The database owns identity and enablement; type-specific JSON must not + // override them. + setBaseConfig(cfg, base) + return cfg, nil +} - return configValue, nil +func newStorageConfig(st storenum.StorageType, base BaseConfig) (StorageConfig, error) { + switch st { + case storenum.Local: + return &LocalStorageConfig{BaseConfig: base}, nil + case storenum.Alist: + return &AlistStorageConfig{BaseConfig: base}, nil + case storenum.Webdav: + return &WebdavStorageConfig{BaseConfig: base}, nil + case storenum.S3: + return &S3StorageConfig{BaseConfig: base}, nil + case storenum.Telegram: + return &TelegramStorageConfig{BaseConfig: base}, nil + case storenum.Rclone: + return &RcloneStorageConfig{BaseConfig: base}, nil + default: + return nil, fmt.Errorf("unsupported storage type: %s", st) } } -func LoadStorageConfigs(v *viper.Viper) ([]StorageConfig, error) { +func setBaseConfig(cfg StorageConfig, base BaseConfig) { + switch typed := cfg.(type) { + case *LocalStorageConfig: + typed.BaseConfig = base + case *AlistStorageConfig: + typed.BaseConfig = base + case *WebdavStorageConfig: + typed.BaseConfig = base + case *S3StorageConfig: + typed.BaseConfig = base + case *TelegramStorageConfig: + typed.BaseConfig = base + case *RcloneStorageConfig: + typed.BaseConfig = base + } +} + +// LoadAllBaseConfigs unmarshals every legacy storage entry, including disabled +// ones. It is only used by the one-time legacy importer. +func LoadAllBaseConfigs(v *viper.Viper) ([]BaseConfig, error) { var baseConfigs []BaseConfig if err := v.UnmarshalKey("storages", &baseConfigs); err != nil { return nil, fmt.Errorf("failed to unmarshal storage configs: %w", err) } - - var configs []StorageConfig - for _, baseCfg := range baseConfigs { - if !baseCfg.Enable { - continue - } - st, err := storenum.ParseStorageType(baseCfg.Type) - if err != nil { - return nil, fmt.Errorf("invalid storage type %s for %s: %w", baseCfg.Type, baseCfg.Name, err) - } - - factory, ok := storageFactories[st] - if !ok { - return nil, fmt.Errorf("unsupported storage type: %s", baseCfg.Type) - } - - cfg, err := factory(&baseCfg) - if err != nil { - return nil, fmt.Errorf("failed to create storage config for %s: %w", baseCfg.Name, err) - } - - if err := cfg.Validate(); err != nil { - return nil, fmt.Errorf("invalid storage config for %s: %w", baseCfg.Name, err) - } - - configs = append(configs, cfg) - } - - return configs, nil + return baseConfigs, nil } diff --git a/config/storage/minio.go b/config/storage/minio.go deleted file mode 100644 index 8e9cd20a..00000000 --- a/config/storage/minio.go +++ /dev/null @@ -1,41 +0,0 @@ -package storage - -import ( - "fmt" - - storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" -) - -type MinioStorageConfig struct { - BaseConfig - Endpoint string `toml:"endpoint" mapstructure:"endpoint" json:"endpoint"` - AccessKeyID string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"` - SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"` - BucketName string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"` - UseSSL bool `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"` - BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"` -} - -func (m *MinioStorageConfig) Validate() error { - if m.Endpoint == "" { - return fmt.Errorf("endpoint is required for minio storage") - } - if m.AccessKeyID == "" || m.SecretAccessKey == "" { - return fmt.Errorf("access_key_id and secret_access_key are required for minio storage") - } - if m.BucketName == "" { - return fmt.Errorf("bucket_name is required for minio storage") - } - if m.BasePath == "" { - return fmt.Errorf("base_path is required for minio storage") - } - return nil -} - -func (m *MinioStorageConfig) GetType() storenum.StorageType { - return storenum.Minio -} - -func (m *MinioStorageConfig) GetName() string { - return m.Name -} diff --git a/config/storage/s3.go b/config/storage/s3.go index 31e7a921..6df39ed7 100644 --- a/config/storage/s3.go +++ b/config/storage/s3.go @@ -12,7 +12,6 @@ type S3StorageConfig struct { AccessKeyID string `toml:"access_key_id" mapstructure:"access_key_id" json:"access_key_id"` SecretAccessKey string `toml:"secret_access_key" mapstructure:"secret_access_key" json:"secret_access_key"` BucketName string `toml:"bucket_name" mapstructure:"bucket_name" json:"bucket_name"` - UseSSL bool `toml:"use_ssl" mapstructure:"use_ssl" json:"use_ssl"` BasePath string `toml:"base_path" mapstructure:"base_path" json:"base_path"` Region string `toml:"region" mapstructure:"region" json:"region"` VirtualHost bool `toml:"virtual_host" mapstructure:"virtual_host" json:"virtual_host"` diff --git a/config/storage/telegram.go b/config/storage/telegram.go index c897f800..a64e9d81 100644 --- a/config/storage/telegram.go +++ b/config/storage/telegram.go @@ -3,6 +3,7 @@ package storage import ( "fmt" + "github.com/gotd/td/constant" storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" ) @@ -19,12 +20,21 @@ type TelegramStorageConfig struct { // only effective when SkipLarge is false // use zip when splitting non-video files or when lossless video splitting is disabled/unavailable SplitSizeMB int64 `toml:"split_size_mb" mapstructure:"split_size_mb" json:"split_size_mb"` + // Use metadata text as the caption when the live source caption is absent. + UseMetadataCaption bool `toml:"use_metadata_caption" mapstructure:"use_metadata_caption" json:"use_metadata_caption"` + // Use grouped_id from metadata to preserve source media groups when the live + // source group key is absent. + UseMetadataAlbum bool `toml:"use_metadata_album" mapstructure:"use_metadata_album" json:"use_metadata_album"` } func (m *TelegramStorageConfig) Validate() error { if m.ChatID == 0 { return fmt.Errorf("chat_id is required for telegram storage") } + peerID := constant.TDLibPeerID(m.ChatID) + if !peerID.IsUser() && !peerID.IsChat() && !peerID.IsChannel() { + return fmt.Errorf("chat_id must be a Bot API style user or chat ID") + } if m.RateLimit < 0 || m.RateBurst < 0 { return fmt.Errorf("rate_limit and rate_burst must be greater than 0 for telegram storage") } diff --git a/config/storage/types.go b/config/storage/types.go index 73d3bdc5..06915abb 100644 --- a/config/storage/types.go +++ b/config/storage/types.go @@ -6,13 +6,17 @@ import ( type StorageConfig interface { Validate() error + GetID() uint GetType() storenum.StorageType GetName() string } type BaseConfig struct { + ID uint `toml:"-" mapstructure:"-" json:"-"` Name string `toml:"name" mapstructure:"name" json:"name"` Type string `toml:"type" mapstructure:"type" json:"type"` Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` RawConfig map[string]any `toml:"-" mapstructure:",remain"` } + +func (b BaseConfig) GetID() uint { return b.ID } diff --git a/config/storage/validate.go b/config/storage/validate.go new file mode 100644 index 00000000..c84e0b62 --- /dev/null +++ b/config/storage/validate.go @@ -0,0 +1,29 @@ +package storage + +import ( + "bytes" + "encoding/json" + "fmt" + + storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" +) + +// ValidateJSON rejects unknown or mistyped keys before a storage record is +// persisted. Keeping this check at the boundary makes the stored JSON a +// versioned, typed contract instead of an arbitrary map. +func ValidateJSON(typ string, raw []byte) error { + st, err := storenum.ParseStorageType(typ) + if err != nil { + return err + } + cfg, err := newStorageConfig(st, BaseConfig{}) + if err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(cfg); err != nil { + return fmt.Errorf("invalid %s storage config: %w", st, err) + } + return nil +} diff --git a/config/tg.go b/config/tg.go index 9b26e403..4b5973a1 100644 --- a/config/tg.go +++ b/config/tg.go @@ -1,21 +1,24 @@ package config type telegramConfig struct { - Token string `toml:"token" mapstructure:"token"` + Token string `toml:"token" mapstructure:"token" json:"token"` AppID int `toml:"app_id" mapstructure:"app_id" json:"app_id"` AppHash string `toml:"app_hash" mapstructure:"app_hash" json:"app_hash"` - Proxy tgProxyConfig `toml:"proxy" mapstructure:"proxy"` + Proxy tgProxyConfig `toml:"proxy" mapstructure:"proxy" json:"proxy"` RpcRetry int `toml:"rpc_retry" mapstructure:"rpc_retry" json:"rpc_retry"` Userbot userbotConfig `toml:"userbot" mapstructure:"userbot" json:"userbot"` MediaGroupTimeout int `toml:"media_group_timeout" mapstructure:"media_group_timeout" json:"media_group_timeout"` + RelayChatID int64 `toml:"relay_chat_id" mapstructure:"relay_chat_id" json:"relay_chat_id"` + RelayKeepMessages bool `toml:"relay_keep_messages" mapstructure:"relay_keep_messages" json:"relay_keep_messages"` } type userbotConfig struct { - Enable bool `toml:"enable" mapstructure:"enable"` - Session string `toml:"session" mapstructure:"session"` + Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` + Session string `toml:"session" mapstructure:"session" json:"session"` + Phone string `toml:"phone" mapstructure:"phone" json:"phone"` } type tgProxyConfig struct { - Enable bool `toml:"enable" mapstructure:"enable"` - URL string `toml:"url" mapstructure:"url"` + Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` + URL string `toml:"url" mapstructure:"url" json:"url"` } diff --git a/config/user.go b/config/user.go index ac97e0e2..57fe5b37 100644 --- a/config/user.go +++ b/config/user.go @@ -2,34 +2,97 @@ package config import ( "github.com/duke-git/lancet/v2/slice" + + "github.com/krau/SaveAny-Bot/pkg/enums/storagefilter" ) -type userConfig struct { - ID int64 `toml:"id" mapstructure:"id" json:"id"` // telegram user id - Storages []string `toml:"storages" mapstructure:"storages" json:"storages"` // storage names - Blacklist bool `toml:"blacklist" mapstructure:"blacklist" json:"blacklist"` // 黑名单模式, storage names 中的存储将不会被使用, 默认为白名单模式 +type UserConfig struct { + ID int64 `toml:"id" mapstructure:"id" json:"id"` // telegram user id + Enabled bool `toml:"enabled" mapstructure:"enabled" json:"enabled"` + DefaultStorageID uint `toml:"default_storage_id" mapstructure:"default_storage_id" json:"default_storage_id"` + StorageFilterIDs []uint `toml:"storage_filter_ids" mapstructure:"storage_filter_ids" json:"storage_filter_ids"` + StorageFilterNames []string `toml:"storages" mapstructure:"storages" json:"-"` // legacy config import only + StorageFilterMode storagefilter.Mode `toml:"storage_filter_mode" mapstructure:"storage_filter_mode" json:"storage_filter_mode"` + // LegacyBlacklist is only used while importing a v0 config file. New + // database-backed settings always use StorageFilterMode. + LegacyBlacklist bool `toml:"blacklist" mapstructure:"blacklist" json:"-"` } -var userIDs []int64 -var storages []string -var userStorages = make(map[int64][]string) +// FilterMode resolves the storage filter mode, including the v0 blacklist +// flag used during config import. +func (u UserConfig) FilterMode() storagefilter.Mode { + if u.StorageFilterMode.Valid() { + return u.StorageFilterMode + } + if u.LegacyBlacklist { + return storagefilter.Blacklist + } + return storagefilter.Whitelist +} -func (c Config) GetStorageNamesByUserID(userID int64) []string { - us, ok := userStorages[userID] - if ok { - return us +func (u UserConfig) filterIDs(c Config) []uint { + if len(u.StorageFilterIDs) > 0 || len(u.StorageFilterNames) == 0 { + return u.StorageFilterIDs + } + ids := make([]uint, 0, len(u.StorageFilterNames)) + for _, name := range u.StorageFilterNames { + if id := c.GetStorageIDByName(name); id != 0 { + ids = append(ids, id) + } + } + return ids +} + +// GetStorageIDsByUserID returns the storage IDs accessible to a user. +func (c Config) GetStorageIDsByUserID(userID int64) []uint { + for _, u := range c.Users { + if u.ID != userID || !u.Enabled { + continue + } + selected := u.filterIDs(c) + if u.FilterMode() == storagefilter.Whitelist { + return selected + } + selectedSet := make(map[uint]struct{}, len(selected)) + for _, id := range selected { + selectedSet[id] = struct{}{} + } + ids := make([]uint, 0, len(c.Storages)) + for _, st := range c.Storages { + if _, denied := selectedSet[st.GetID()]; !denied { + ids = append(ids, st.GetID()) + } + } + return ids } return nil } +func (c Config) GetStorageNamesByUserID(userID int64) []string { + ids := c.GetStorageIDsByUserID(userID) + names := make([]string, 0, len(ids)) + for _, id := range ids { + if name := c.GetStorageNameByID(id); name != "" { + names = append(names, name) + } + } + return names +} + func (c Config) GetUsersID() []int64 { - return userIDs + ids := make([]int64, 0, len(c.Users)) + for _, u := range c.Users { + if u.Enabled { + ids = append(ids, u.ID) + } + } + return ids +} + +func (c Config) HasStorageID(userID int64, storageID uint) bool { + return slice.Contain(c.GetStorageIDsByUserID(userID), storageID) } func (c Config) HasStorage(userID int64, storageName string) bool { - us, ok := userStorages[userID] - if !ok { - return false - } - return slice.Contain(us, storageName) + return slice.Contain(c.GetStorageNamesByUserID(userID), storageName) } diff --git a/config/viper.go b/config/viper.go deleted file mode 100644 index ad5b0fe2..00000000 --- a/config/viper.go +++ /dev/null @@ -1,236 +0,0 @@ -package config - -import ( - "context" - "fmt" - "net" - "net/http" - "net/url" - "strings" - "time" - - "github.com/charmbracelet/log" - "github.com/duke-git/lancet/v2/slice" - "github.com/krau/SaveAny-Bot/config/storage" - "github.com/spf13/viper" - "golang.org/x/net/proxy" -) - -type Config struct { - Lang string `toml:"lang" mapstructure:"lang" json:"lang"` - Workers int `toml:"workers" mapstructure:"workers"` - Retry int `toml:"retry" mapstructure:"retry"` - NoCleanCache bool `toml:"no_clean_cache" mapstructure:"no_clean_cache" json:"no_clean_cache"` - Threads int `toml:"threads" mapstructure:"threads" json:"threads"` - Stream bool `toml:"stream" mapstructure:"stream" json:"stream"` - Proxy string `toml:"proxy" mapstructure:"proxy" json:"proxy"` - Log logConfig `toml:"log" mapstructure:"log" json:"log"` - Aria2 aria2Config `toml:"aria2" mapstructure:"aria2" json:"aria2"` - API apiConfig `toml:"api" mapstructure:"api" json:"api"` - - Cache cacheConfig `toml:"cache" mapstructure:"cache" json:"cache"` - Users []userConfig `toml:"users" mapstructure:"users" json:"users"` - Temp tempConfig `toml:"temp" mapstructure:"temp"` - DB dbConfig `toml:"db" mapstructure:"db"` - Telegram telegramConfig `toml:"telegram" mapstructure:"telegram"` - Storages []storage.StorageConfig `toml:"-" mapstructure:"-" json:"storages"` - Parser parserConfig `toml:"parser" mapstructure:"parser" json:"parser"` - Hook hookConfig `toml:"hook" mapstructure:"hook" json:"hook"` - Ytdlp YtdlpConfig `toml:"ytdlp" mapstructure:"ytdlp" json:"ytdlp"` -} - -type aria2Config struct { - Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` - Url string `toml:"url" mapstructure:"url" json:"url"` - Secret string `toml:"secret" mapstructure:"secret" json:"secret"` - KeepFile bool `toml:"keep_file" mapstructure:"keep_file" json:"keep_file"` -} - -type apiConfig struct { - Enable bool `toml:"enable" mapstructure:"enable" json:"enable"` - Host string `toml:"host" mapstructure:"host" json:"host"` - Port int `toml:"port" mapstructure:"port" json:"port"` - Token string `toml:"token" mapstructure:"token" json:"token"` -} - -var cfg = &Config{} - -func C() Config { - return *cfg -} - -func (c Config) GetStorageByName(name string) storage.StorageConfig { - for _, storage := range c.Storages { - if storage.GetName() == name { - return storage - } - } - return nil -} - -func Init(ctx context.Context, configFile ...string) error { - logger := log.FromContext(ctx) - - // Reset side tables for re-init. - storages = nil - userIDs = nil - userStorages = make(map[int64][]string) - - viper.SetConfigType("toml") - viper.SetEnvPrefix("SAVEANY") - viper.AutomaticEnv() - replacer := strings.NewReplacer(".", "_") - viper.SetEnvKeyReplacer(replacer) - - // 配置文件支持传入一个 http(s) URL 地址 - loadedFromURL := false - if len(configFile) > 0 && configFile[0] != "" { - cfg := configFile[0] - if strings.HasPrefix(cfg, "http://") || strings.HasPrefix(cfg, "https://") { - client := &http.Client{Timeout: 30 * time.Second} - resp, err := client.Get(cfg) - if err != nil { - return fmt.Errorf("failed to fetch remote config file: %w", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to fetch remote config file: status code %d", resp.StatusCode) - } - if err := viper.ReadConfig(resp.Body); err != nil { - return fmt.Errorf("failed to read remote config file: %w", err) - } - loadedFromURL = true - } else { - viper.SetConfigFile(cfg) - } - } else { - viper.SetConfigName("config") - viper.AddConfigPath(".") - viper.AddConfigPath("/etc/saveany/") - } - - defaultConfigs := map[string]any{ - "lang": "zh-Hans", - "workers": 3, - "retry": 3, - "threads": 4, - "log.level": "debug", - - "cache.ttl": 86400, - "cache.num_counters": 1e5, - "cache.max_cost": 1e6, - - "telegram.app_id": 1025907, - "telegram.app_hash": "452b0359b988148995f22ff0f4229750", - "telegram.rpc_retry": 5, - "telegram.userbot.enable": false, - "telegram.userbot.session": "data/usersession.db", - - "temp.base_path": "cache/", - - "db.path": "data/saveany.db", - "db.session": "data/session.db", - - "api.enable": false, - "api.host": "0.0.0.0", - "api.port": 8080, - "api.token": "", - - "ytdlp.recode": "mp4", - "ytdlp.filename_template": DefaultYtdlpFilenameTemplate, - "ytdlp.max_height": 0, - "ytdlp.format": "", - "ytdlp.restrict_filenames": false, - } - - for key, value := range defaultConfigs { - viper.SetDefault(key, value) - } - - if !loadedFromURL { - if err := viper.ReadInConfig(); err != nil { - logger.Errorf("Error reading config file: %v", err) - return err - } - } - - if err := viper.Unmarshal(cfg); err != nil { - logger.Errorf("Error unmarshalling config file: %v", err) - return err - } - - storagesConfig, err := storage.LoadStorageConfigs(viper.GetViper()) - if err != nil { - return fmt.Errorf("error loading storage configs: %w", err) - } - cfg.Storages = storagesConfig - - storageNames := make(map[string]struct{}) - for _, storage := range cfg.Storages { - if _, ok := storageNames[storage.GetName()]; ok { - return fmt.Errorf("duplicate storage name: %s", storage.GetName()) - } - storageNames[storage.GetName()] = struct{}{} - } - - if cfg.Workers < 1 { - cfg.Workers = 1 - } - if cfg.Threads < 1 { - cfg.Threads = 1 - } - if cfg.Retry < 1 { - cfg.Retry = 1 - } - - for _, storage := range cfg.Storages { - storages = append(storages, storage.GetName()) - } - for _, user := range cfg.Users { - userIDs = append(userIDs, user.ID) - if user.Blacklist { - userStorages[user.ID] = slice.Compact(slice.Difference(storages, user.Storages)) - } else { - userStorages[user.ID] = user.Storages - } - } - if cfg.Proxy != "" { - http.DefaultTransport, err = newProxyTransport(cfg.Proxy) - if err != nil { - return fmt.Errorf("failed to create proxy transport: %w", err) - } - } - return nil -} - -func newProxyTransport(proxyStr string) (*http.Transport, error) { - proxyURL, err := url.Parse(proxyStr) - if err != nil { - return nil, err - } - transport := &http.Transport{ - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - switch proxyURL.Scheme { - case "http", "https": - transport.Proxy = http.ProxyURL(proxyURL) - - case "socks5", "socks5h": - dialer, err := proxy.FromURL(proxyURL, proxy.Direct) - if err != nil { - return nil, err - } - transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - return dialer.(proxy.ContextDialer).DialContext(ctx, network, addr) - } - - default: - return nil, fmt.Errorf("unsupported proxy type: %s", proxyURL.Scheme) - } - - return transport, nil -} diff --git a/core/core.go b/core/core.go index e10d0862..6574f6a9 100644 --- a/core/core.go +++ b/core/core.go @@ -2,19 +2,29 @@ package core import ( "context" + "encoding/json" "errors" + "fmt" "sync" + "time" "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/taskstatus" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" "github.com/krau/SaveAny-Bot/pkg/queue" "github.com/krau/SaveAny-Bot/pkg/taskevent" + "github.com/krau/SaveAny-Bot/pkg/taskspec" ) var ( queueOnce sync.Once queueInstance *queue.TaskQueue[Executable] + workerWG sync.WaitGroup + + taskSinkFactoryMu sync.RWMutex + taskSinkFactory func(Executable) taskevent.Sink ) // initQueue lazily creates the shared task queue. @@ -25,6 +35,12 @@ func initQueue() *queue.TaskQueue[Executable] { return queueInstance } +const ( + TaskSourceBot = "bot" + TaskSourceAPI = "api" + TaskSourceRetry = "retry" +) + type Executable interface { Type() tasktype.TaskType Title() string @@ -32,43 +48,144 @@ type Executable interface { Execute(ctx context.Context) error } +// DestinationInfo lets task records include the storage target for every +// creation path, including tasks started from Telegram handlers. +type DestinationInfo interface { + StorageName() string + StoragePath() string +} + +// ParamSource exposes the creation parameters that a queued task needs to be +// rebuilt after a restart. +type ParamSource interface { + TaskParams() (json.RawMessage, error) +} + func worker(ctx context.Context, qe *queue.TaskQueue[Executable], semaphore chan struct{}) { + defer workerWG.Done() logger := log.FromContext(ctx) execHooks := config.C().Hook.Exec for { semaphore <- struct{}{} qtask, err := qe.Get() if err != nil { - logger.Error("Failed to get task from queue:", err) + if !errors.Is(err, queue.ErrQueueClosed) { + logger.Error("Failed to get task from queue:", err) + } break // queue closed and empty } exe := qtask.Data taskCtx := qtask.Context() + recordCtx := context.WithoutCancel(taskCtx) logger.Infof("Processing task: %s", exe.TaskID()) taskevent.Emit(taskCtx, taskevent.Event{TaskID: exe.TaskID(), Phase: taskevent.PhaseStart}) if err := ExecCommandString(taskCtx, execHooks.TaskBeforeStart); err != nil { logger.Errorf("Failed to execute before start hook for task %s: %v", exe.TaskID(), err) } + startedAt := time.Now() + if dbErr := database.UpdateTaskRecord(recordCtx, exe.TaskID(), map[string]any{ + "status": taskstatus.Running.String(), + "started_at": startedAt, + }); dbErr != nil { + logger.Errorf("Failed to update task record for %s: %v", exe.TaskID(), dbErr) + } err = exe.Execute(taskCtx) + items := collectItemResults(exe) + failedItems := countItems(items, ItemStatusFailed) + status := taskstatus.Completed.String() if err != nil { if errors.Is(taskCtx.Err(), context.Canceled) { - logger.Infof("Task %s was canceled", exe.TaskID()) - if err := ExecCommandString(ctx, execHooks.TaskCancel); err != nil { - logger.Errorf("Failed to execute cancel hook for task %s: %v", exe.TaskID(), err) + current, dbErr := database.GetTaskRecord(recordCtx, exe.TaskID()) + if dbErr == nil && current.Status == taskstatus.Cancelled.String() { + status = taskstatus.Cancelled.String() + logger.Infof("Task %s was canceled", exe.TaskID()) + if err := ExecCommandString(ctx, execHooks.TaskCancel); err != nil { + logger.Errorf("Failed to execute cancel hook for task %s: %v", exe.TaskID(), err) + } + } else { + // Process shutdown cancels task contexts; leave the record queued + // so the next start can resume it from persisted cache. + status = taskstatus.Queued.String() + logger.Infof("Task %s interrupted; it will resume after restart", exe.TaskID()) } } else { + status = taskstatus.Failed.String() logger.Errorf("Failed to execute task %s: %v", exe.TaskID(), err) if err := ExecCommandString(ctx, execHooks.TaskFail); err != nil { logger.Errorf("Failed to execute fail hook for task %s: %v", exe.TaskID(), err) } } + } else if failedItems > 0 { + status = taskstatus.Partial.String() + logger.Warnf("Task %s finished with %d failed files", exe.TaskID(), failedItems) + if err := ExecCommandString(ctx, execHooks.TaskFail); err != nil { + logger.Errorf("Failed to execute fail hook for task %s: %v", exe.TaskID(), err) + } } else { logger.Infof("Task %s completed successfully", exe.TaskID()) if err := ExecCommandString(ctx, execHooks.TaskSuccess); err != nil { logger.Errorf("Failed to execute success hook for task %s: %v", exe.TaskID(), err) } } - taskevent.Emit(taskCtx, taskevent.Event{TaskID: exe.TaskID(), Phase: taskevent.PhaseDone, Err: err}) + if len(items) > 0 && status != taskstatus.Queued.String() { + records := make([]database.TaskItemRecord, 0, len(items)) + for _, item := range items { + records = append(records, database.TaskItemRecord{ + Index: item.Index, + Name: item.Name, + Path: item.Path, + StorageName: item.StorageName, + StorageID: item.StorageID, + Status: string(item.Status), + Error: item.Error, + Size: item.Size, + SourceChatID: item.SourceChatID, + SourceMsgID: item.SourceMessageID, + }) + } + if dbErr := database.ReplaceTaskItems(recordCtx, exe.TaskID(), records); dbErr != nil { + logger.Errorf("Failed to store task items for %s: %v", exe.TaskID(), dbErr) + } + metadataRecords := make([]database.TaskItemMetadata, 0, len(items)) + for _, item := range items { + if item.MetadataJSON == "" { + continue + } + metadataRecords = append(metadataRecords, database.TaskItemMetadata{ + ItemIndex: item.Index, + Version: item.MetadataVersion, + SourceChatID: item.SourceChatID, + SourceMessageID: item.SourceMessageID, + JSON: item.MetadataJSON, + }) + } + if dbErr := database.ReplaceTaskItemMetadata(recordCtx, exe.TaskID(), metadataRecords); dbErr != nil { + logger.Errorf("Failed to store task metadata for %s: %v", exe.TaskID(), dbErr) + } + } + recordFields := map[string]any{"status": status} + if status == taskstatus.Queued.String() { + recordFields["started_at"] = nil + recordFields["finished_at"] = nil + recordFields["error"] = "" + } else { + recordFields["finished_at"] = time.Now() + } + if status == taskstatus.Partial.String() { + recordFields["error"] = partialError(items) + } else if err != nil && status != taskstatus.Cancelled.String() { + recordFields["error"] = err.Error() + } + if dbErr := database.UpdateTaskRecord(recordCtx, exe.TaskID(), recordFields); dbErr != nil { + logger.Errorf("Failed to finish task record for %s: %v", exe.TaskID(), dbErr) + } + taskevent.Emit(taskCtx, taskevent.Event{ + TaskID: exe.TaskID(), + Phase: taskevent.PhaseDone, + Err: err, + Partial: failedItems > 0 && err == nil, + Cancelled: status == taskstatus.Cancelled.String(), + }) qe.Done(qtask.ID) <-semaphore } @@ -78,10 +195,11 @@ func Run(ctx context.Context) { log.FromContext(ctx).Info("Start processing tasks...") semaphore := make(chan struct{}, config.C().Workers) q := initQueue() - for range config.C().Workers { + workers := config.C().Workers + workerWG.Add(workers) + for range workers { go worker(ctx, q, semaphore) } - } // Close stops the queue and unblocks workers in Get. @@ -91,19 +209,180 @@ func Close() { } } +// WaitWorkers waits up to timeout for all worker goroutines to exit. +func WaitWorkers(timeout time.Duration) bool { + done := make(chan struct{}) + go func() { + workerWG.Wait() + close(done) + }() + select { + case <-done: + return true + case <-time.After(timeout): + return false + } +} + func AddTask(ctx context.Context, task Executable) error { + return AddTaskWithSource(ctx, task, TaskSourceBot) +} + +// SetTaskSinkFactory installs the optional observer used for tasks started +// outside the API. The API task path registers its own sink with webhook data. +func SetTaskSinkFactory(factory func(Executable) taskevent.Sink) { + taskSinkFactoryMu.Lock() + taskSinkFactory = factory + taskSinkFactoryMu.Unlock() +} + +func taskSinkFor(task Executable, source string) taskevent.Sink { + if source == TaskSourceAPI { + return nil + } + taskSinkFactoryMu.RLock() + factory := taskSinkFactory + taskSinkFactoryMu.RUnlock() + if factory == nil { + return nil + } + return factory(task) +} + +// AddTaskWithSource records the task before enqueueing it so every execution +// has a matching database row. +func AddTaskWithSource(ctx context.Context, task Executable, source string) error { + logger := log.FromContext(ctx) + sink := taskSinkFor(task, source) + if sink != nil { + ctx = taskevent.WithSink(ctx, sink) + } + record := &database.TaskRecord{ + TaskID: task.TaskID(), + Type: task.Type().String(), + Title: task.Title(), + Status: taskstatus.Queued.String(), + Source: source, + ParamsVersion: taskspec.Version, + } + if destination, ok := task.(DestinationInfo); ok { + record.StorageName = destination.StorageName() + record.StorageID = config.C().GetStorageIDByName(record.StorageName) + record.Path = destination.StoragePath() + } + if source, ok := task.(ParamSource); ok { + params, err := source.TaskParams() + if err != nil { + logger.Warnf("Failed to serialize params for task %s: %v", task.TaskID(), err) + } else { + record.Params = string(params) + } + } + if err := database.CreateTaskRecord(ctx, record); err != nil { + logger.Errorf("Failed to create task record for %s: %v", task.TaskID(), err) + } + if err := initQueue().Add(queue.NewTask(ctx, task.TaskID(), task.Title(), task)); err != nil { + if sink != nil { + sink.Emit(taskevent.Event{TaskID: task.TaskID(), Phase: taskevent.PhaseDone, Err: err}) + } + if dbErr := database.DeleteTaskRecord(ctx, task.TaskID()); dbErr != nil { + logger.Errorf("Failed to remove task record for %s: %v", task.TaskID(), dbErr) + } + return err + } + return nil +} + +func EnrichTaskRecord(ctx context.Context, taskID, storageName, storagePath string) error { + return database.UpdateTaskRecord(ctx, taskID, map[string]any{ + "storage_name": storageName, + "storage_id": config.C().GetStorageIDByName(storageName), + "path": storagePath, + }) +} + +// IsTaskRunning reports whether the task is still being executed, including +// the window between cancellation and the worker persisting its results. +func IsTaskRunning(id string) bool { + return initQueue().IsRunning(id) +} + +// HasTask reports whether the task is known to the queue, queued or running. +func HasTask(id string) bool { + return initQueue().Has(id) +} + +// EnqueueExisting adds a rebuilt task without creating another record. The +// progress sink factory is attached so the web console still tracks it. +func EnqueueExisting(ctx context.Context, task Executable) error { + if sink := taskSinkFor(task, TaskSourceBot); sink != nil { + ctx = taskevent.WithSink(ctx, sink) + } return initQueue().Add(queue.NewTask(ctx, task.TaskID(), task.Title(), task)) } func CancelTask(ctx context.Context, id string) error { - err := queueInstance.CancelTask(id) - return err + q := initQueue() + if q.Has(id) { + task, _ := q.GetTask(id) + // Persist the cancellation before canceling the context so the worker + // can distinguish it from a process shutdown and does not requeue it. + if dbErr := database.UpdateTaskRecord(context.WithoutCancel(ctx), id, map[string]any{ + "status": taskstatus.Cancelled.String(), + "finished_at": time.Now(), + }); dbErr != nil { + log.FromContext(ctx).Errorf("Failed to cancel task record for %s: %v", id, dbErr) + } + err := q.CancelTask(id) + if err == nil && task != nil { + taskevent.Emit(task.Context(), taskevent.Event{ + TaskID: id, + Phase: taskevent.PhaseDone, + Cancelled: true, + }) + } + return err + } + // A queued record may not be in the queue yet while restart recovery waits + // for a Telegram client. + record, err := database.GetTaskRecord(ctx, id) + if err != nil { + return fmt.Errorf("task %s does not exist", id) + } + if isTerminalTaskStatus(record.Status) { + return fmt.Errorf("task %s is not active", id) + } + if dbErr := database.UpdateTaskRecord(context.WithoutCancel(ctx), id, map[string]any{ + "status": taskstatus.Cancelled.String(), + "finished_at": time.Now(), + }); dbErr != nil { + return dbErr + } + if q.Has(id) { + return q.CancelTask(id) + } + return nil +} + +func isTerminalTaskStatus(status string) bool { + return taskstatus.Status(status).IsTerminal() } func GetLength(ctx context.Context) int { + if queueInstance == nil { + return 0 + } return queueInstance.ActiveLength() } +// ActiveTaskCount includes queued and running tasks regardless of cancellation. +func ActiveTaskCount() int { + if queueInstance == nil { + return 0 + } + return queueInstance.ActiveCount() +} + func GetRunningTasks(ctx context.Context) []queue.TaskInfo { return queueInstance.RunningTasks() } diff --git a/core/result.go b/core/result.go new file mode 100644 index 00000000..67f881de --- /dev/null +++ b/core/result.go @@ -0,0 +1,59 @@ +package core + +import "fmt" + +// ItemStatus is the final state of one file inside a task. +type ItemStatus string + +const ( + ItemStatusCompleted ItemStatus = "completed" + ItemStatusFailed ItemStatus = "failed" + ItemStatusSkipped ItemStatus = "skipped" + ItemStatusCancelled ItemStatus = "cancelled" +) + +// ItemResult describes the outcome of one file in a task. Tasks that handle +// multiple files implement ResultReporter so the outcome survives after the +// task leaves memory. +type ItemResult struct { + Index int + Name string + Path string + StorageName string + StorageID uint + Status ItemStatus + Error string + Size int64 + SourceChatID int64 + SourceMessageID int + MetadataJSON string + MetadataVersion int +} + +// ResultReporter exposes per-file outcomes after execution. +type ResultReporter interface { + ItemResults() []ItemResult +} + +func collectItemResults(task Executable) []ItemResult { + reporter, ok := task.(ResultReporter) + if !ok { + return nil + } + return reporter.ItemResults() +} + +func countItems(items []ItemResult, status ItemStatus) int { + count := 0 + for _, item := range items { + if item.Status == status { + count++ + } + } + return count +} + +func partialError(items []ItemResult) string { + failed := countItems(items, ItemStatusFailed) + return fmt.Sprintf("%d of %d files failed", failed, len(items)) +} diff --git a/core/resume/resume.go b/core/resume/resume.go new file mode 100644 index 00000000..227f7c11 --- /dev/null +++ b/core/resume/resume.go @@ -0,0 +1,429 @@ +// Package resume rebuilds tasks that were queued or running when the process +// stopped. Task creation parameters are persisted with the task record, so a +// task can be re-enqueued without resubmitting the original request. +package resume + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/client/bot" + userclient "github.com/krau/SaveAny-Bot/client/user" + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/core/tasks/aria2dl" + "github.com/krau/SaveAny-Bot/core/tasks/batchtfile" + "github.com/krau/SaveAny-Bot/core/tasks/directlinks" + "github.com/krau/SaveAny-Bot/core/tasks/parsed" + "github.com/krau/SaveAny-Bot/core/tasks/telegraph" + "github.com/krau/SaveAny-Bot/core/tasks/tfile" + "github.com/krau/SaveAny-Bot/core/tasks/transfer" + "github.com/krau/SaveAny-Bot/core/tasks/ytdlp" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/aria2" + storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage" + "github.com/krau/SaveAny-Bot/pkg/enums/taskstatus" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/storagetypes" + "github.com/krau/SaveAny-Bot/pkg/taskspec" + tphclient "github.com/krau/SaveAny-Bot/pkg/telegraph" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/ext" +) + +const retryInterval = 15 * time.Second + +// ErrNeedsClient means the task needs a Telegram client that is not running +// yet. The record stays queued and is retried later. +var ErrNeedsClient = errors.New("telegram client not available") + +// ErrAlreadyDone means every file in the task was uploaded before the +// process stopped. The record only needs to be marked completed. +var ErrAlreadyDone = errors.New("task already completed") + +var scanMu sync.Mutex + +// Run resumes queued tasks and keeps retrying while some of them are waiting +// for a Telegram client. +func Run(ctx context.Context) { + for { + pending := runOnce(ctx) + if pending == 0 { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(retryInterval): + } + } +} + +func runOnce(ctx context.Context) (pending int) { + scanMu.Lock() + defer scanMu.Unlock() + + logger := log.FromContext(ctx).WithPrefix("resume") + records, err := database.ListQueuedTaskRecords(ctx) + if err != nil { + logger.Errorf("Failed to list queued tasks: %v", err) + return 1 + } + for i := range records { + taskID := records[i].TaskID + if core.HasTask(taskID) { + continue + } + record, err := database.GetTaskRecord(ctx, taskID) + if err != nil || record.Status != taskstatus.Queued.String() { + continue + } + task, taskCtx, err := rebuild(ctx, record) + switch { + case errors.Is(err, ErrNeedsClient): + pending++ + logger.Debugf("Task %s waits for a telegram client", taskID) + continue + case errors.Is(err, ErrAlreadyDone): + if dbErr := database.UpdateTaskRecord(ctx, taskID, map[string]any{ + "status": taskstatus.Completed.String(), + "error": "", + "finished_at": time.Now(), + }); dbErr != nil { + logger.Errorf("Failed to mark completed task %s: %v", taskID, dbErr) + } else { + logger.Infof("Task %s was already completed before restart", taskID) + } + continue + case err != nil: + logger.Warnf("Cannot resume task %s: %v", taskID, err) + if dbErr := database.UpdateTaskRecord(ctx, taskID, map[string]any{ + "status": taskstatus.Failed.String(), + "error": "cannot resume: " + err.Error(), + "finished_at": time.Now(), + }); dbErr != nil { + logger.Errorf("Failed to mark task %s as failed: %v", taskID, dbErr) + } + continue + } + if err := core.EnqueueExisting(taskCtx, task); err != nil { + logger.Errorf("Failed to enqueue resumed task %s: %v", taskID, err) + pending++ + continue + } + // The task may have been cancelled while it was being rebuilt. + if current, err := database.GetTaskRecord(ctx, taskID); err == nil && current.Status == taskstatus.Cancelled.String() { + if err := core.CancelTask(ctx, taskID); err != nil { + logger.Errorf("Failed to cancel resumed task %s: %v", taskID, err) + } + continue + } + logger.Infof("Resumed queued task %s", taskID) + } + return pending +} + +func rebuild(ctx context.Context, record *database.TaskRecord) (core.Executable, context.Context, error) { + if record.Params == "" { + return nil, ctx, errors.New("missing task parameters") + } + if record.ParamsVersion > taskspec.Version { + return nil, ctx, fmt.Errorf("unsupported task parameter version %d", record.ParamsVersion) + } + switch tasktype.TaskType(record.Type) { + case tasktype.TaskTypeDirectlinks: + var spec taskspec.DirectLinks + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + stor, err := resolveStorage(ctx, record.StorageName, record.StorageID) + if err != nil { + return nil, ctx, err + } + taskCtx, err := attachTelegramClient(ctx, stor) + if err != nil { + return nil, ctx, err + } + target := strings.TrimSuffix(record.Path, "/") + if spec.Path != nil { + target = *spec.Path + } + return directlinks.NewTask(record.TaskID, taskCtx, spec.URLs, stor, target, nil), taskCtx, nil + case tasktype.TaskTypeYtdlp: + var spec taskspec.YTDLP + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + stor, err := resolveStorage(ctx, record.StorageName, record.StorageID) + if err != nil { + return nil, ctx, err + } + taskCtx, err := attachTelegramClient(ctx, stor) + if err != nil { + return nil, ctx, err + } + return ytdlp.NewTask(record.TaskID, taskCtx, spec.URLs, spec.Flags, stor, record.Path, nil), taskCtx, nil + case tasktype.TaskTypeAria2: + var spec taskspec.Aria2 + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + stor, err := resolveStorage(ctx, record.StorageName, record.StorageID) + if err != nil { + return nil, ctx, err + } + taskCtx, err := attachTelegramClient(ctx, stor) + if err != nil { + return nil, ctx, err + } + cfg := config.C().Aria2 + if !cfg.Enable { + return nil, ctx, errors.New("aria2 is not enabled") + } + aria2Client, err := aria2.NewClient(cfg.Url, cfg.Secret) + if err != nil { + return nil, ctx, fmt.Errorf("failed to create aria2 client: %w", err) + } + gid := spec.GID + if gid == "" { + gid, err = aria2Client.AddURI(ctx, spec.URIs, nil) + if err != nil { + return nil, ctx, fmt.Errorf("failed to add aria2 task: %w", err) + } + } else if _, err := aria2Client.TellStatus(ctx, gid); err != nil { + gid, err = aria2Client.AddURI(ctx, spec.URIs, nil) + if err != nil { + return nil, ctx, fmt.Errorf("failed to add aria2 task: %w", err) + } + } + return aria2dl.NewTask(record.TaskID, taskCtx, gid, spec.URIs, aria2Client, stor, record.Path, nil), taskCtx, nil + case tasktype.TaskTypeParseditem: + var spec taskspec.Parsed + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + if spec.Item == nil { + return nil, ctx, errors.New("missing parsed item") + } + stor, err := resolveStorage(ctx, record.StorageName, record.StorageID) + if err != nil { + return nil, ctx, err + } + taskCtx, err := attachTelegramClient(ctx, stor) + if err != nil { + return nil, ctx, err + } + return parsed.NewTask(record.TaskID, taskCtx, stor, record.Path, spec.Item, nil), taskCtx, nil + case tasktype.TaskTypeTphpics: + var spec taskspec.Telegraph + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + if len(spec.Pics) == 0 { + return nil, ctx, errors.New("missing telegraph images") + } + stor, err := resolveStorage(ctx, record.StorageName, record.StorageID) + if err != nil { + return nil, ctx, err + } + taskCtx, err := attachTelegramClient(ctx, stor) + if err != nil { + return nil, ctx, err + } + return telegraph.NewTask(record.TaskID, taskCtx, spec.PhPath, spec.Pics, stor, record.Path, tphclient.NewClient(), nil), taskCtx, nil + case tasktype.TaskTypeTransfer: + var spec taskspec.Transfer + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + elems := make([]transfer.TaskElement, 0, len(spec.Items)) + storages := make([]storage.Storage, 0, len(spec.Items)*2) + for _, item := range spec.Items { + sourceStor, err := resolveStorage(ctx, item.SourceStorage, item.SourceStorageID) + if err != nil { + return nil, ctx, err + } + targetStor, err := resolveStorage(ctx, item.TargetStorage, item.TargetStorageID) + if err != nil { + return nil, ctx, err + } + elem := transfer.NewTaskElement(sourceStor, storagetypes.FileInfo{ + Name: item.Name, + Path: item.SourcePath, + Size: item.Size, + ModTime: item.ModTime, + }, targetStor, item.TargetPath) + elems = append(elems, *elem) + storages = append(storages, sourceStor, targetStor) + } + if len(elems) == 0 { + return nil, ctx, errors.New("missing transfer files") + } + taskCtx, err := attachTelegramClient(ctx, storages...) + if err != nil { + return nil, ctx, err + } + return transfer.NewTransferTask(record.TaskID, taskCtx, elems, nil, true), taskCtx, nil + case tasktype.TaskTypeTgfiles: + return rebuildTG(ctx, record) + default: + return nil, ctx, fmt.Errorf("unsupported task type %q", record.Type) + } +} + +func rebuildTG(ctx context.Context, record *database.TaskRecord) (core.Executable, context.Context, error) { + var spec taskspec.TGFile + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return nil, ctx, fmt.Errorf("invalid parameters: %w", err) + } + if len(spec.Items) == 0 { + return nil, ctx, errors.New("missing telegram file items") + } + done := make(map[string]struct{}, len(spec.Done)) + for _, id := range spec.Done { + done[id] = struct{}{} + } + remaining := 0 + for _, item := range spec.Items { + if _, ok := done[item.ID]; item.ID == "" || !ok { + remaining++ + } + } + if remaining == 0 { + return nil, ctx, ErrAlreadyDone + } + candidates := make([]*ext.Context, 0, 2) + for _, candidate := range []*ext.Context{bot.ExtContext(), userclient.GetCtx()} { + if candidate != nil { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + return nil, ctx, ErrNeedsClient + } + elems := make([]batchtfile.TaskElement, 0, len(spec.Items)) + var taskExt *ext.Context + for _, item := range spec.Items { + if _, ok := done[item.ID]; item.ID != "" && ok { + continue + } + stor, err := resolveStorage(ctx, item.Storage, item.StorageID) + if err != nil { + return nil, ctx, err + } + message, clientCtx, err := tgutil.GetMessageByIDFreshAny(candidates, item.ChatID, item.MessageID) + if err != nil { + if clientPending() { + return nil, ctx, ErrNeedsClient + } + return nil, ctx, fmt.Errorf("failed to fetch source message %d: %w", item.MessageID, err) + } + if message.Media == nil { + return nil, ctx, fmt.Errorf("source message %d has no media", item.MessageID) + } + file, err := tfilepkg.FromMediaMessage(message.Media, clientCtx.Raw, message) + if err != nil { + return nil, ctx, fmt.Errorf("failed to rebuild file %s: %w", item.Name, err) + } + file.SetName(item.Name) + var elem *batchtfile.TaskElement + if item.ID != "" { + elem, err = batchtfile.NewTaskElementWithID(item.ID, stor, item.Path, file) + } else { + elem, err = batchtfile.NewTaskElement(stor, item.Path, file) + } + if err != nil { + return nil, ctx, fmt.Errorf("failed to create task element for %s: %w", item.Name, err) + } + elems = append(elems, *elem) + if taskExt == nil { + taskExt = clientCtx + } + } + taskCtx := tgutil.ExtWithContext(ctx, taskExt) + if spec.Overwrite { + taskCtx = storage.WithOverwrite(taskCtx) + } + if len(spec.Items) == 1 { + task, err := tfile.NewTGFileTask(record.TaskID, taskCtx, elems[0].File, elems[0].Storage, elems[0].Path, nil) + if err != nil { + return nil, ctx, err + } + task.ElementID = elems[0].ID + task.Done = spec.Done + return task, taskCtx, nil + } + task := batchtfile.NewBatchTGFileTask(record.TaskID, taskCtx, elems, nil, true) + task.Done = spec.Done + return task, taskCtx, nil +} + +// activeClient returns the first running Telegram client. +func activeClient() *ext.Context { + if botCtx := bot.ExtContext(); botCtx != nil { + return botCtx + } + return userclient.GetCtx() +} + +// attachTelegramClient adds a client context when the task writes to Telegram. +// A Telegram destination without a running client must wait for one. +func attachTelegramClient(ctx context.Context, storages ...storage.Storage) (context.Context, error) { + needsTelegram := false + for _, stor := range storages { + if stor != nil && stor.Type() == storenum.Telegram { + needsTelegram = true + break + } + } + client := activeClient() + if client == nil { + if needsTelegram { + return ctx, ErrNeedsClient + } + return ctx, nil + } + return tgutil.ExtWithContext(ctx, client), nil +} + +// clientPending reports whether a configured Telegram client is still starting +// or not logged in, so a fetch failure should be retried instead of failing +// the task permanently. +func clientPending() bool { + cfg := config.C().Telegram + if cfg.Token != "" && bot.CurrentStatus().State != "running" { + return true + } + if cfg.Userbot.Enable && userclient.CurrentStatus().State != "running" { + return true + } + return false +} + +func resolveStorage(ctx context.Context, name string, id uint) (storage.Storage, error) { + if id != 0 { + stor, err := storage.GetStorageByID(ctx, id) + if err == nil { + return stor, nil + } + if name == "" { + return nil, err + } + } + if name == "" { + return nil, errors.New("storage is empty") + } + stor, ok := storage.GetStorage(name) + if !ok { + return nil, fmt.Errorf("storage %q is not available", name) + } + return stor, nil +} diff --git a/core/retry/retry.go b/core/retry/retry.go new file mode 100644 index 00000000..0b47b622 --- /dev/null +++ b/core/retry/retry.go @@ -0,0 +1,172 @@ +// Package retry re-creates a task from its persisted item records, so files +// that failed (for example because the disk was full) can be downloaded again +// without resubmitting the original message. +package retry + +import ( + "context" + "errors" + "fmt" + "path" + + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/core/tasks/batchtfile" + "github.com/krau/SaveAny-Bot/core/tasks/tfile" + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/taskstatus" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" + "github.com/krau/mygotg/ext" + "github.com/rs/xid" +) + +var ( + ErrTaskActive = errors.New("task is still active") + ErrNothingToRetry = errors.New("task has no failed files to retry") + ErrUnsupportedType = errors.New("retry is only supported for telegram file tasks") + ErrStorageMissing = errors.New("target storage is no longer available") + ErrSourceUnavailable = errors.New("failed file has no source message to retry") + ErrClientUnavailable = errors.New("no telegram client available") +) + +// FailedTask enqueues a new task containing the failed (or unfinished) files +// of taskID and returns the new task ID. Messages are fetched fresh so their +// file references are current. extCtxs are tried in order (bot first, then +// userbot) to match the client that can see the source chat. +func FailedTask(ctx context.Context, extCtxs []*ext.Context, taskID string) (string, int, error) { + record, err := database.GetTaskRecord(ctx, taskID) + if err != nil { + return "", 0, fmt.Errorf("task not found: %w", err) + } + if core.IsTaskRunning(taskID) { + return "", 0, ErrTaskActive + } + if !isTerminalStatus(record.Status) { + return "", 0, ErrTaskActive + } + if record.Type != tasktype.TaskTypeTgfiles.String() { + return "", 0, ErrUnsupportedType + } + items, err := database.ListTaskItems(ctx, taskID) + if err != nil { + return "", 0, fmt.Errorf("failed to load task items: %w", err) + } + selected := retryableItems(items) + if len(selected) == 0 { + return "", 0, ErrNothingToRetry + } + candidates := make([]*ext.Context, 0, len(extCtxs)) + for _, candidate := range extCtxs { + if candidate != nil { + candidates = append(candidates, candidate) + } + } + if len(candidates) == 0 { + return "", 0, ErrClientUnavailable + } + + elems := make([]batchtfile.TaskElement, 0, len(selected)) + var taskExt *ext.Context + for _, item := range selected { + if item.SourceChatID == 0 || item.SourceMsgID == 0 { + return "", 0, fmt.Errorf("%w: %s", ErrSourceUnavailable, item.Name) + } + msg, clientCtx, err := tgutil.GetMessageByIDFreshAny(candidates, item.SourceChatID, item.SourceMsgID) + if err != nil { + return "", 0, fmt.Errorf("%w: %s: %v", ErrSourceUnavailable, item.Name, err) + } + if taskExt == nil { + taskExt = clientCtx + } + if msg.Media == nil { + return "", 0, fmt.Errorf("%w: %s", ErrSourceUnavailable, item.Name) + } + stor, err := itemStorage(ctx, item, record) + if err != nil { + return "", 0, err + } + file, err := tfilepkg.FromMediaMessage(msg.Media, clientCtx.Raw, msg) + if err != nil { + return "", 0, fmt.Errorf("failed to rebuild file %s: %w", item.Name, err) + } + file.SetName(item.Name) + target := item.Path + if target == "" { + target = path.Join(record.Path, item.Name) + } + elem, err := batchtfile.NewTaskElement(stor, target, file) + if err != nil { + return "", 0, fmt.Errorf("failed to create task element for %s: %w", item.Name, err) + } + elems = append(elems, *elem) + } + + newID := xid.New().String() + // A failed item may have left a partial file at its destination; the retry + // must replace it instead of writing a renamed duplicate. + taskCtx := storage.WithOverwrite(tgutil.ExtWithContext(ctx, taskExt)) + var task core.Executable + if len(elems) == 1 { + single, err := tfile.NewTGFileTask(newID, taskCtx, elems[0].File, elems[0].Storage, elems[0].Path, nil) + if err != nil { + return "", 0, fmt.Errorf("failed to create retry task: %w", err) + } + task = single + } else { + task = batchtfile.NewBatchTGFileTask(newID, taskCtx, elems, nil, true) + } + if err := core.AddTaskWithSource(taskCtx, task, core.TaskSourceRetry); err != nil { + return "", 0, fmt.Errorf("failed to enqueue retry task: %w", err) + } + return newID, len(elems), nil +} + +func itemStorage(ctx context.Context, item database.TaskItemRecord, record *database.TaskRecord) (storage.Storage, error) { + id := item.StorageID + if id == 0 { + id = record.StorageID + } + if id != 0 { + if stor, err := storage.GetStorageByID(ctx, id); err == nil { + return stor, nil + } + } + name := item.StorageName + if name == "" { + name = record.StorageName + } + stor, ok := storage.GetStorage(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrStorageMissing, name) + } + return stor, nil +} + +// retryableItems prefers failed files; if a task failed before per-item +// outcomes were recorded, unfinished items are retried instead. +func retryableItems(items []database.TaskItemRecord) []database.TaskItemRecord { + failed := make([]database.TaskItemRecord, 0, len(items)) + for _, item := range items { + if item.Status == string(core.ItemStatusFailed) { + failed = append(failed, item) + } + } + if len(failed) > 0 { + return failed + } + for _, item := range items { + switch item.Status { + case string(core.ItemStatusCompleted), string(core.ItemStatusSkipped): + continue + default: + failed = append(failed, item) + } + } + return failed +} + +func isTerminalStatus(status string) bool { + return taskstatus.Status(status).IsTerminal() +} diff --git a/core/task_params.go b/core/task_params.go new file mode 100644 index 00000000..38f0b386 --- /dev/null +++ b/core/task_params.go @@ -0,0 +1,50 @@ +package core + +import ( + "context" + "encoding/json" + "errors" + "sync" + + "github.com/krau/SaveAny-Bot/database" + "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var taskParamsMu sync.Mutex + +// MarkTaskElementDone records an uploaded Telegram file element in the task's +// persisted parameters so a restart can skip it. +func MarkTaskElementDone(ctx context.Context, taskID, elementID string) error { + if elementID == "" { + return errors.New("empty task element ID") + } + taskParamsMu.Lock() + defer taskParamsMu.Unlock() + + record, err := database.GetTaskRecord(ctx, taskID) + if err != nil { + return err + } + if record.Type != tasktype.TaskTypeTgfiles.String() { + return errors.New("task is not a telegram file task") + } + if record.Params == "" { + return errors.New("task has no persisted parameters") + } + var spec taskspec.TGFile + if err := json.Unmarshal([]byte(record.Params), &spec); err != nil { + return err + } + for _, id := range spec.Done { + if id == elementID { + return nil + } + } + spec.Done = append(spec.Done, elementID) + raw, err := json.Marshal(spec) + if err != nil { + return err + } + return database.UpdateTaskRecord(ctx, taskID, map[string]any{"params": string(raw)}) +} diff --git a/core/tasks/aria2dl/execute.go b/core/tasks/aria2dl/execute.go index 1dfdd492..1739f241 100644 --- a/core/tasks/aria2dl/execute.go +++ b/core/tasks/aria2dl/execute.go @@ -49,7 +49,9 @@ func (t *Task) Execute(ctx context.Context) error { t.Progress.OnDone(ctx, t, nil) } - if _, err := t.Aria2Client.RemoveDownloadResult(context.Background(), t.GID); err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := t.Aria2Client.RemoveDownloadResult(cleanupCtx, t.GID); err != nil { logger.Warnf("Failed to remove aria2 download result: %v", err) } diff --git a/core/tasks/aria2dl/params.go b/core/tasks/aria2dl/params.go new file mode 100644 index 00000000..2f257de1 --- /dev/null +++ b/core/tasks/aria2dl/params.go @@ -0,0 +1,14 @@ +package aria2dl + +import ( + "encoding/json" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + return json.Marshal(taskspec.Aria2{GID: t.GID, URIs: t.URIs}) +} diff --git a/core/tasks/aria2dl/progress.go b/core/tasks/aria2dl/progress.go index 6a254675..37e5b076 100644 --- a/core/tasks/aria2dl/progress.go +++ b/core/tasks/aria2dl/progress.go @@ -56,23 +56,27 @@ func (p *Progress) OnStart(ctx context.Context, task *Task) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(task.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(task.TaskID())) ext.EditMessage(p.chatID, req) } // OnProgress implements ProgressTracker. func (p *Progress) OnProgress(ctx context.Context, task *Task, status *aria2.Status) { - totalLength, _ := strconv.ParseInt(status.TotalLength, 10, 64) - completedLength, _ := strconv.ParseInt(status.CompletedLength, 10, 64) - downloadSpeed, _ := strconv.ParseInt(status.DownloadSpeed, 10, 64) + totalLength, err := strconv.ParseInt(status.TotalLength, 10, 64) + if err != nil { + log.FromContext(ctx).Warnf("Invalid aria2 total length %q: %v", status.TotalLength, err) + return + } + completedLength, err := strconv.ParseInt(status.CompletedLength, 10, 64) + if err != nil { + log.FromContext(ctx).Warnf("Invalid aria2 completed length %q: %v", status.CompletedLength, err) + return + } + downloadSpeed, err := strconv.ParseInt(status.DownloadSpeed, 10, 64) + if err != nil { + log.FromContext(ctx).Warnf("Invalid aria2 download speed %q: %v", status.DownloadSpeed, err) + return + } if totalLength == 0 { return @@ -109,15 +113,7 @@ func (p *Progress) OnProgress(ctx context.Context, task *Task, status *aria2.Sta } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(task.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(task.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.chatID, req) diff --git a/core/tasks/aria2dl/task.go b/core/tasks/aria2dl/task.go index 0e09be16..01e88d14 100644 --- a/core/tasks/aria2dl/task.go +++ b/core/tasks/aria2dl/task.go @@ -11,6 +11,7 @@ import ( ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { ID string @@ -28,6 +29,17 @@ func (t *Task) Title() string { return fmt.Sprintf("[%s](Aria2 GID:%s->%s:%s)", t.Type(), t.GID, t.Storage.Name(), t.StorPath) } +func (t *Task) StorageName() string { + if t.Storage == nil { + return "" + } + return t.Storage.Name() +} + +func (t *Task) StoragePath() string { + return t.StorPath +} + // Type implements core.Executable. func (t *Task) Type() tasktype.TaskType { return tasktype.TaskTypeAria2 diff --git a/core/tasks/aria2dl/task_test.go b/core/tasks/aria2dl/task_test.go index 7a0728ea..0811932d 100644 --- a/core/tasks/aria2dl/task_test.go +++ b/core/tasks/aria2dl/task_test.go @@ -34,8 +34,8 @@ func (m *mockStorage) Save(ctx context.Context, reader io.Reader, path string) e return nil } -func (m *mockStorage) Exists(ctx context.Context, path string) bool { - return false +func (m *mockStorage) Exists(ctx context.Context, path string) (bool, error) { + return false, nil } func (m *mockStorage) JoinStoragePath(path string) string { diff --git a/core/tasks/batchtfile/execute.go b/core/tasks/batchtfile/execute.go index 99dd51c0..0a744098 100644 --- a/core/tasks/batchtfile/execute.go +++ b/core/tasks/batchtfile/execute.go @@ -16,6 +16,7 @@ import ( "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/common/utils/ioutil" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" "github.com/krau/SaveAny-Bot/pkg/storagetypes" "github.com/krau/SaveAny-Bot/pkg/taskevent" @@ -87,7 +88,8 @@ func (t *Task) executionGroups() []executionGroup { for i := 0; i < len(t.elems); { elem := &t.elems[i] batchSaver, batchCapable := elem.Storage.(storage.StorageBatchSaver) - if !batchCapable || elem.sourceGroupKey == "" { + key := elem.executionGroupKey() + if !batchCapable || key == "" { groups = append(groups, executionGroup{elems: []*TaskElement{elem}}) i++ continue @@ -96,7 +98,7 @@ func (t *Task) executionGroups() []executionGroup { end := i + 1 for end < len(t.elems) { next := &t.elems[end] - if next.Storage != elem.Storage || next.sourceGroupKey != elem.sourceGroupKey { + if next.Storage != elem.Storage || next.executionGroupKey() != key { break } end++ @@ -120,7 +122,7 @@ func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error return err } defer t.unmarkProcessing(elem.ID) - err := t.processElement(gctx, *elem) + err := t.processElement(gctx, elem) if err != nil && t.IgnoreErrors && !errors.Is(err, context.Canceled) { // Per-item failure: keep siblings running. log.FromContext(ctx).Warnf("Element %s failed (ignored): %v", elem.ID, err) @@ -133,8 +135,9 @@ func (t *Task) processElements(ctx context.Context, elems []*TaskElement) error } func (t *Task) processBatch(ctx context.Context, group executionGroup) error { + var cleanup []*TaskElement defer func() { - for _, elem := range group.elems { + for _, elem := range cleanup { if err := os.Remove(elem.localPath); err != nil && !os.IsNotExist(err) { log.FromContext(ctx).Warnf("Failed to cleanup batch cache file %s: %v", elem.localPath, err) } @@ -206,18 +209,29 @@ func (t *Task) processBatch(ctx context.Context, group executionGroup) error { } openFiles = append(openFiles, file) items = append(items, storagetypes.BatchItem{ - Reader: file, - StoragePath: elem.Path, - Size: stat.Size(), - SourceGroupKey: elem.sourceGroupKey, - Caption: elem.sourceCaption, - PreserveCaption: elem.preserveCaption, + Reader: file, + StoragePath: elem.Path, + Size: stat.Size(), + SourceGroupKey: elem.sourceGroupKey, + Caption: elem.sourceCaption, + PreserveCaption: elem.preserveCaption, + MetadataJSON: elem.metadataJSON, + MetadataGroupKey: elem.metadataGroupKey, }) } for index, item := range items { t.recordDownloadComplete(successElems[index].ID, item.Size) } - return t.saveBatchItems(ctx, successElems, items) + if err := t.saveBatchItems(ctx, successElems, items); err != nil { + return err + } + for _, elem := range successElems { + if err := core.MarkTaskElementDone(ctx, t.ID, elem.ID); err != nil { + return fmt.Errorf("failed to persist element completion: %w", err) + } + } + cleanup = successElems + return nil } func (t *Task) saveBatchItems(ctx context.Context, successElems []*TaskElement, items []storagetypes.BatchItem) error { @@ -287,37 +301,37 @@ func (t *Task) unmarkProcessing(id string) { func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error { logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name())) logger.Info("Starting file download") - localFile, err := fsutil.CreateFile(elem.localPath) + if err := tdler.DownloadToCache(ctx, elem.File, elem.localPath, func(w io.WriterAt) io.WriterAt { + return ioutil.NewProgressWriterAt(w, func(n int) { + t.recordItemDownload(elem.ID, int64(n), time.Now()) + downloaded := t.downloaded.Add(int64(n)) + t.notifyProgress(ctx) + taskevent.Emit(ctx, taskevent.Event{ + TaskID: t.ID, + Phase: taskevent.PhaseProgress, + TotalBytes: t.totalSize, + DownloadedBytes: downloaded, + }) + }) + }); err != nil { + t.markItemFailed(ctx, elem.ID, FailureStageDownload, err) + t.notifyStateChange(ctx) + return fmt.Errorf("failed to download file: %w", err) + } + logger.Info("File downloaded successfully") + fileStat, err := os.Stat(elem.localPath) if err != nil { t.markItemFailed(ctx, elem.ID, FailureStageCache, err) t.notifyStateChange(ctx) - return fmt.Errorf("failed to create local file: %w", err) - } - wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) { - t.recordItemDownload(elem.ID, int64(n), time.Now()) - downloaded := t.downloaded.Add(int64(n)) - t.notifyProgress(ctx) - taskevent.Emit(ctx, taskevent.Event{ - TaskID: t.ID, - Phase: taskevent.PhaseProgress, - TotalBytes: t.totalSize, - DownloadedBytes: downloaded, - }) - }) - _, downloadErr := tdler.NewDownloader(elem.File).Parallel(ctx, wrAt) - closeErr := localFile.Close() - if downloadErr != nil { - t.markItemFailed(ctx, elem.ID, FailureStageDownload, downloadErr) - t.notifyStateChange(ctx) - return fmt.Errorf("failed to download file: %w", downloadErr) + return fmt.Errorf("failed to get file stat: %w", err) } - if closeErr != nil { - t.markItemFailed(ctx, elem.ID, FailureStageCache, closeErr) + if elem.File.Size() > 0 && fileStat.Size() == 0 { + err := fmt.Errorf("downloaded file is empty (expected %d bytes)", elem.File.Size()) + t.markItemFailed(ctx, elem.ID, FailureStageDownload, err) t.notifyStateChange(ctx) - return fmt.Errorf("failed to close cache file: %w", closeErr) + return err } - logger.Info("File downloaded successfully") - if path.Ext(elem.FileName()) == "" { + if path.Ext(elem.FileName()) == "" && path.Ext(elem.Path) == "" { if ext := fsutil.DetectFileExt(elem.localPath); ext != "" { elem.Path += ext } @@ -327,14 +341,20 @@ func (t *Task) downloadElement(ctx context.Context, elem *TaskElement) error { return nil } -func (t *Task) processElement(ctx context.Context, elem TaskElement) error { +func (t *Task) processElement(ctx context.Context, elem *TaskElement) error { logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", elem.File.Name())) if elem.stream { pr, pw := io.Pipe() defer pr.Close() errg, uploadCtx := errgroup.WithContext(ctx) + uploadCtx = storagetypes.WithMessageMetadata(uploadCtx, elem.metadataJSON) + if elem.preserveCaption { + uploadCtx = storagetypes.WithSourceCaption(uploadCtx, elem.sourceCaption) + } + savedPath := elem.Path errg.Go(func() error { - err := elem.Storage.Save(uploadCtx, pr, elem.Path) + var err error + savedPath, err = storage.SaveWithPath(uploadCtx, elem.Storage, pr, elem.Path) if err != nil { t.markItemFailed(uploadCtx, elem.ID, FailureStageUpload, err) t.notifyStateChange(ctx) @@ -355,7 +375,11 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error { errg.Go(func() error { defer pw.Close() logger.Info("Starting file download in stream mode") - _, err := tdler.NewDownloader(elem.File).Stream(uploadCtx, wr) + downloader, cleanup := tdler.NewDownloader(uploadCtx, elem.File) + if cleanup != nil { + defer cleanup(uploadCtx) + } + _, err := downloader.Stream(uploadCtx, wr) if err != nil { logger.Errorf("Failed to download file: %v", err) t.markItemFailed(uploadCtx, elem.ID, FailureStageDownload, err) @@ -374,35 +398,30 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error { }) t.recordDownloadComplete(elem.ID, streamedBytes) t.markItemCompleted(elem.ID) + if err := core.MarkTaskElementDone(ctx, t.ID, elem.ID); err != nil { + t.markItemFailed(ctx, elem.ID, FailureStageUpload, err) + t.notifyStateChange(ctx) + return fmt.Errorf("failed to persist element completion: %w", err) + } + elem.saveMetadataSidecar(ctx, savedPath) t.notifyStateChange(ctx) logger.Info("File downloaded successfully in stream mode") return nil } logger.Info("Starting file download") - localFile, err := fsutil.CreateFile(elem.localPath) - if err != nil { - t.markItemFailed(ctx, elem.ID, FailureStageCache, err) - t.notifyStateChange(ctx) - return fmt.Errorf("failed to create local file: %w", err) - } - defer func() { - if err := localFile.CloseAndRemove(); err != nil { - logger.Errorf("Failed to close local file: %v", err) - } - }() - wrAt := ioutil.NewProgressWriterAt(localFile, func(n int) { - t.recordItemDownload(elem.ID, int64(n), time.Now()) - downloaded := t.downloaded.Add(int64(n)) - t.notifyProgress(ctx) - taskevent.Emit(ctx, taskevent.Event{ - TaskID: t.ID, - Phase: taskevent.PhaseProgress, - TotalBytes: t.totalSize, - DownloadedBytes: downloaded, + if err := tdler.DownloadToCache(ctx, elem.File, elem.localPath, func(w io.WriterAt) io.WriterAt { + return ioutil.NewProgressWriterAt(w, func(n int) { + t.recordItemDownload(elem.ID, int64(n), time.Now()) + downloaded := t.downloaded.Add(int64(n)) + t.notifyProgress(ctx) + taskevent.Emit(ctx, taskevent.Event{ + TaskID: t.ID, + Phase: taskevent.PhaseProgress, + TotalBytes: t.totalSize, + DownloadedBytes: downloaded, + }) }) - }) - _, err = tdler.NewDownloader(elem.File).Parallel(ctx, wrAt) - if err != nil { + }); err != nil { t.markItemFailed(ctx, elem.ID, FailureStageDownload, err) t.notifyStateChange(ctx) return fmt.Errorf("failed to download file: %w", err) @@ -414,20 +433,30 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error { elem.Path = elem.Path + ext } } - var fileStat os.FileInfo - fileStat, err = os.Stat(elem.localPath) + fileStat, err := os.Stat(elem.localPath) if err != nil { t.markItemFailed(ctx, elem.ID, FailureStageCache, err) t.notifyStateChange(ctx) return fmt.Errorf("failed to get file stat: %w", err) } + if elem.File.Size() > 0 && fileStat.Size() == 0 { + err := fmt.Errorf("downloaded file is empty (expected %d bytes)", elem.File.Size()) + t.markItemFailed(ctx, elem.ID, FailureStageDownload, err) + t.notifyStateChange(ctx) + return err + } t.recordDownloadComplete(elem.ID, fileStat.Size()) vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size()) + vctx = storagetypes.WithMessageMetadata(vctx, elem.metadataJSON) + if elem.preserveCaption { + vctx = storagetypes.WithSourceCaption(vctx, elem.sourceCaption) + } t.startUpload(vctx) onProgress := t.uploadCallback(vctx, elem.ID) attempt := 0 retryLimit := int(config.C().Retry) lastFailureStage := FailureStageUpload + savedPath := elem.Path err = retry.Retry(func() error { attempt++ var file *os.File @@ -442,8 +471,9 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error { onProgress(0, fileStat.Size()) if progressSaver, ok := elem.Storage.(storage.StorageProgressSaver); ok { err = progressSaver.SaveWithProgress(vctx, file, elem.Path, onProgress) + savedPath = elem.Path } else { - err = elem.Storage.Save(vctx, ioutil.NewProgressReader(file, fileStat.Size(), onProgress), elem.Path) + savedPath, err = storage.SaveWithPath(vctx, elem.Storage, ioutil.NewProgressReader(file, fileStat.Size(), onProgress), elem.Path) } if err != nil { logger.Errorf("Failed to save file: %s, retrying...", err) @@ -457,7 +487,16 @@ func (t *Task) processElement(ctx context.Context, elem TaskElement) error { if err == nil { onProgress(fileStat.Size(), fileStat.Size()) t.markItemCompleted(elem.ID) + if err := core.MarkTaskElementDone(vctx, t.ID, elem.ID); err != nil { + t.markItemFailed(vctx, elem.ID, FailureStageUpload, err) + t.notifyStateChange(vctx) + return fmt.Errorf("failed to persist element completion: %w", err) + } + elem.saveMetadataSidecar(vctx, savedPath) t.notifyStateChange(vctx) + if err := os.Remove(elem.localPath); err != nil { + logger.Errorf("Failed to remove cache file: %v", err) + } } else { t.markItemFailed(vctx, elem.ID, lastFailureStage, err) t.notifyStateChange(vctx) diff --git a/core/tasks/batchtfile/params.go b/core/tasks/batchtfile/params.go new file mode 100644 index 00000000..bbe6d7fc --- /dev/null +++ b/core/tasks/batchtfile/params.go @@ -0,0 +1,41 @@ +package batchtfile + +import ( + "encoding/json" + "errors" + + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" + "github.com/krau/SaveAny-Bot/pkg/taskspec" + "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + items := make([]taskspec.TGFileItem, 0, len(t.elems)) + for i := range t.elems { + elem := &t.elems[i] + messageFile, ok := elem.File.(tfile.TGFileMessage) + if !ok || messageFile.Message() == nil { + return nil, errors.New("task element has no source message") + } + message := messageFile.Message() + items = append(items, taskspec.TGFileItem{ + ID: elem.ID, + ChatID: tgutil.ChatIdFromPeer(message.GetPeerID()), + MessageID: message.GetID(), + Name: elem.File.Name(), + Storage: elem.Storage.Name(), + StorageID: storage.ID(elem.Storage), + Path: elem.Path, + }) + } + overwrite := false + if t.ctx != nil { + overwrite, _ = t.ctx.Value(ctxkey.OverwriteExisting).(bool) + } + return json.Marshal(taskspec.TGFile{Items: items, Overwrite: overwrite, Done: t.Done}) +} diff --git a/core/tasks/batchtfile/progress.go b/core/tasks/batchtfile/progress.go index a74649bd..1eebee30 100644 --- a/core/tasks/batchtfile/progress.go +++ b/core/tasks/batchtfile/progress.go @@ -123,9 +123,7 @@ func buildBatchEditMessageRequest(messageID int, taskID string, message rendered req.SetEntities(message.Entities) } if cancellable { - req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{ - Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)}, - }}}) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(taskID)) } return req } @@ -191,12 +189,21 @@ func buildBatchDoneMarkup(ctx context.Context, info TaskInfo, skipped []string, } if err == nil { completed, _, _, failed := itemCounts(items) - // Report per-element failures instead of full completion. - totalSkipped := len(skipped) + failed - if totalSkipped > 0 { + // Keep real failures out of the skipped bucket so partial results are + // never reported as a full success. + if failed > 0 { + return localizedProgressMarkup(i18nk.BotMsgProgressBatchDoneWithFailed, map[string]any{ + "Success": completed, + "Failed": failed, + "Skipped": len(skipped), + "Size": dlutil.FormatSize(totalSize), + "List": failedItemsList(items, 5), + }) + } + if len(skipped) > 0 { return localizedProgressMarkup(i18nk.BotMsgProgressBatchDoneWithSkipped, map[string]any{ "Success": completed, - "Skipped": totalSkipped, + "Skipped": len(skipped), "Size": dlutil.FormatSize(totalSize), }) } @@ -450,6 +457,38 @@ func firstError(items []TaskItemProgress) string { return "" } +func failedItemsList(items []TaskItemProgress, limit int) string { + failed := make([]TaskItemProgress, 0, len(items)) + for _, item := range items { + if item.Phase == ItemPhaseFailed { + failed = append(failed, item) + } + } + if len(failed) == 0 { + return "" + } + var builder strings.Builder + builder.WriteString("\n\n") + for i, item := range failed { + if i >= limit { + builder.WriteString(fmt.Sprintf("\n… +%d", len(failed)-limit)) + break + } + builder.WriteString("\n❌ ") + builder.WriteString(item.Name) + if stage := failureStageLabel(item.FailureStage); stage != "" { + builder.WriteString(" (") + builder.WriteString(stage) + builder.WriteString(")") + } + if reason := truncateRunes(item.Error, 60); reason != "" { + builder.WriteString(": ") + builder.WriteString(reason) + } + } + return builder.String() +} + func failureStageLabel(stage FailureStage) string { switch stage { case FailureStageDownload: diff --git a/core/tasks/batchtfile/results.go b/core/tasks/batchtfile/results.go new file mode 100644 index 00000000..5a9c3f94 --- /dev/null +++ b/core/tasks/batchtfile/results.go @@ -0,0 +1,85 @@ +package batchtfile + +import ( + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ResultReporter = (*Task)(nil) + +// ItemResults reports the final state of every element. The worker persists +// these rows so failed files stay visible and retryable after the task ends. +func (t *Task) ItemResults() []core.ItemResult { + states := t.Items() + byID := make(map[string]TaskItemProgress, len(states)) + for _, state := range states { + byID[state.ID] = state + } + + results := make([]core.ItemResult, 0, len(t.elems)) + for i := range t.elems { + elem := &t.elems[i] + state, ok := byID[elem.ID] + result := core.ItemResult{ + Index: i + 1, + Name: elem.File.Name(), + Path: elem.Path, + StorageName: elem.Storage.Name(), + StorageID: storage.ID(elem.Storage), + Size: state.Size, + } + if !ok { + result.Status = core.ItemStatusSkipped + } else { + result.Index = state.Index + switch state.Phase { + case ItemPhaseCompleted: + result.Status = core.ItemStatusCompleted + case ItemPhaseFailed: + result.Status = core.ItemStatusFailed + result.Error = failureSummary(state) + case ItemPhaseStopped: + result.Status = core.ItemStatusCancelled + default: + result.Status = core.ItemStatusSkipped + } + } + if messageFile, ok := elem.File.(tfile.TGFileMessage); ok && messageFile.Message() != nil { + msg := messageFile.Message() + result.SourceChatID = tgutil.ChatIdFromPeer(msg.GetPeerID()) + result.SourceMessageID = msg.GetID() + result.MetadataJSON = elem.metadataJSON + result.MetadataVersion = elem.metadataVersion + } + results = append(results, result) + } + return results +} + +func failureSummary(state TaskItemProgress) string { + stage := failureStageToken(state.FailureStage) + if state.Error == "" { + return stage + } + return stage + ": " + state.Error +} + +// failureStageToken returns a stable, non-localized stage name for storage. +func failureStageToken(stage FailureStage) string { + switch stage { + case FailureStageDownload: + return "download" + case FailureStageCache: + return "local cache" + case FailureStageUpload: + return "upload" + case FailureStageConfirm: + return "remote confirmation" + case FailureStageBatchUpload: + return "batch upload" + default: + return "internal task" + } +} diff --git a/core/tasks/batchtfile/task.go b/core/tasks/batchtfile/task.go index 82c4bbd1..36761541 100644 --- a/core/tasks/batchtfile/task.go +++ b/core/tasks/batchtfile/task.go @@ -2,36 +2,45 @@ package batchtfile import ( "context" + "encoding/json" "fmt" "path/filepath" "sync" "sync/atomic" + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/common/utils/tgutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/metadata" "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" "github.com/rs/xid" ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type TaskElement struct { - ID string - Storage storage.Storage - Path string - File tfile.TGFile - localPath string - stream bool - sourceGroupKey string - sourceCaption string - preserveCaption bool + ID string + Storage storage.Storage + Path string + File tfile.TGFile + localPath string + stream bool + sourceGroupKey string + sourceCaption string + preserveCaption bool + metadataJSON string + metadataVersion int + metadataGroupKey string } type Task struct { ID string + Done []string ctx context.Context elems []TaskElement Progress ProgressTracker @@ -63,11 +72,19 @@ func NewTaskElement( path string, file tfile.TGFile, ) (*TaskElement, error) { - id := xid.New().String() + return NewTaskElementWithID(xid.New().String(), stor, path, file) +} + +func NewTaskElementWithID( + id string, + stor storage.Storage, + path string, + file tfile.TGFile, +) (*TaskElement, error) { groupKey, caption, preserveCaption := sourceMetadata(file) _, ok := stor.(storage.StorageCannotStream) if !config.C().Stream || ok { - cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name()))) + cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, fsutil.SafeBaseName(file.Name())))) if err != nil { return nil, fmt.Errorf("failed to get absolute path for cache: %w", err) } @@ -136,5 +153,58 @@ func NewBatchTGFileTask( IgnoreErrors: ignoreErrors, processingMu: sync.RWMutex{}, } + if config.C().Metadata.Enabled { + for i := range task.elems { + task.elems[i].metadataJSON, task.elems[i].metadataVersion = buildElementMetadata(ctx, task.elems[i].File) + task.elems[i].metadataGroupKey = metadataGroupKey(task.elems[i].metadataJSON) + } + } return task } + +func buildElementMetadata(ctx context.Context, file tfile.TGFile) (string, int) { + messageFile, ok := file.(tfile.TGFileMessage) + if !ok || messageFile.Message() == nil { + return "", 0 + } + metaJSON, version, err := metadata.BuildJSON(ctx, messageFile.Message(), file.Name(), file.Size(), config.C().Metadata.IncludeText) + if err != nil { + log.FromContext(ctx).Warnf("Failed to build task metadata: %v", err) + return "", 0 + } + return metaJSON, version +} + +func (e *TaskElement) saveMetadataSidecar(ctx context.Context, actualPath string) { + if !config.C().Metadata.Sidecar || e.metadataJSON == "" || actualPath == "" { + return + } + if err := storage.WriteMetadataSidecar(ctx, e.Storage, actualPath, []byte(e.metadataJSON)); err != nil { + log.FromContext(ctx).Warnf("Failed to save metadata sidecar for %s: %v", actualPath, err) + } +} + +func metadataGroupKey(raw string) string { + if raw == "" { + return "" + } + var meta struct { + Message struct { + GroupedID int64 `json:"grouped_id"` + Chat struct { + ID int64 `json:"id"` + } `json:"chat"` + } `json:"message"` + } + if err := json.Unmarshal([]byte(raw), &meta); err != nil || meta.Message.GroupedID == 0 { + return "" + } + return fmt.Sprintf("metadata:%d:%d", meta.Message.Chat.ID, meta.Message.GroupedID) +} + +func (e *TaskElement) executionGroupKey() string { + if e.sourceGroupKey != "" { + return e.sourceGroupKey + } + return e.metadataGroupKey +} diff --git a/core/tasks/batchtfile/taskinfo.go b/core/tasks/batchtfile/taskinfo.go index c99d0eff..7b7ce4bf 100644 --- a/core/tasks/batchtfile/taskinfo.go +++ b/core/tasks/batchtfile/taskinfo.go @@ -50,9 +50,25 @@ func (t *Task) Count() int { } func (t *Task) Processing() []TaskElementInfo { + t.processingMu.RLock() + defer t.processingMu.RUnlock() processing := make([]TaskElementInfo, 0, len(t.elems)) for _, elem := range t.processing { processing = append(processing, elem) } return processing } + +func (t *Task) StorageName() string { + if len(t.elems) == 0 { + return "" + } + return t.elems[0].Storage.Name() +} + +func (t *Task) StoragePath() string { + if len(t.elems) == 0 { + return "" + } + return t.elems[0].Path +} diff --git a/core/tasks/directlinks/execute.go b/core/tasks/directlinks/execute.go index f3511fb8..6978fa02 100644 --- a/core/tasks/directlinks/execute.go +++ b/core/tasks/directlinks/execute.go @@ -56,6 +56,7 @@ func (t *Task) Execute(ctx context.Context) error { if file.Name == "" { return fmt.Errorf("failed to determine filename for %s: Content-Disposition header is empty and URL does not contain a valid filename", file.URL) } + file.Name = fsutil.SafeBaseName(file.Name) return nil }) diff --git a/core/tasks/directlinks/params.go b/core/tasks/directlinks/params.go new file mode 100644 index 00000000..b167249b --- /dev/null +++ b/core/tasks/directlinks/params.go @@ -0,0 +1,19 @@ +package directlinks + +import ( + "encoding/json" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + urls := make([]string, 0, len(t.files)) + for _, file := range t.files { + urls = append(urls, file.URL) + } + path := t.StorPath + return json.Marshal(taskspec.DirectLinks{URLs: urls, Path: &path}) +} diff --git a/core/tasks/directlinks/progress.go b/core/tasks/directlinks/progress.go index 708d451e..12e45527 100644 --- a/core/tasks/directlinks/progress.go +++ b/core/tasks/directlinks/progress.go @@ -145,15 +145,7 @@ func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.chatID, req) @@ -187,15 +179,7 @@ func (p *Progress) OnStart(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext.EditMessage(p.chatID, req) } diff --git a/core/tasks/directlinks/task.go b/core/tasks/directlinks/task.go index 350fa4ff..a951ce01 100644 --- a/core/tasks/directlinks/task.go +++ b/core/tasks/directlinks/task.go @@ -28,6 +28,7 @@ func (f *File) FileSize() int64 { } var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { ID string @@ -110,7 +111,12 @@ func NewTask( _, ok := stor.(storage.StorageCannotStream) stream := config.C().Stream && !ok files := make([]*File, 0, len(links)) + seen := make(map[string]struct{}, len(links)) for _, link := range links { + if _, duplicate := seen[link]; duplicate { + continue + } + seen[link] = struct{}{} files = append(files, &File{ URL: link, }) diff --git a/core/tasks/parsed/execute.go b/core/tasks/parsed/execute.go index 18bf3b5c..fd339f32 100644 --- a/core/tasks/parsed/execute.go +++ b/core/tasks/parsed/execute.go @@ -94,11 +94,12 @@ func (t *Task) processResource(ctx context.Context, resource parser.Resource) er } return resp.ContentLength }()) + filename := fsutil.SafeBaseName(resource.Filename) if t.stream { - return t.Stor.Save(ctx, resp.Body, path.Join(t.StorPath, resource.Filename)) + return t.Stor.Save(ctx, resp.Body, path.Join(t.StorPath, filename)) } cacheFile, err := fsutil.CreateFile(filepath.Join(config.C().Temp.BasePath, - fmt.Sprintf("resource_%s_%s", t.ID, resource.Filename))) + fmt.Sprintf("resource_%s_%s", t.ID, filename))) if err != nil { return fmt.Errorf("failed to create cache file for resource %s: %w", resource.URL, err) } @@ -137,7 +138,7 @@ func (t *Task) processResource(ctx context.Context, resource parser.Resource) er if err != nil { return fmt.Errorf("failed to seek cache file for resource %s: %w", resource.URL, err) } - return t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, resource.Filename)) + return t.Stor.Save(ctx, cacheFile, path.Join(t.StorPath, filename)) }, retry.Context(ctx), retry.RetryTimes(uint(config.C().Retry))) if ctx.Err() != nil { return ctx.Err() diff --git a/core/tasks/parsed/params.go b/core/tasks/parsed/params.go new file mode 100644 index 00000000..1c438d85 --- /dev/null +++ b/core/tasks/parsed/params.go @@ -0,0 +1,18 @@ +package parsed + +import ( + "encoding/json" + "errors" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + if t.item == nil { + return nil, errors.New("task has no parsed item") + } + return json.Marshal(taskspec.Parsed{Item: t.item}) +} diff --git a/core/tasks/parsed/progress.go b/core/tasks/parsed/progress.go index 8c2b0aed..a732e09d 100644 --- a/core/tasks/parsed/progress.go +++ b/core/tasks/parsed/progress.go @@ -57,15 +57,7 @@ func (p *Progress) OnStart(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.ChatID, req) @@ -116,15 +108,7 @@ func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.ChatID, req) diff --git a/core/tasks/parsed/task.go b/core/tasks/parsed/task.go index d52e88a5..dc268329 100644 --- a/core/tasks/parsed/task.go +++ b/core/tasks/parsed/task.go @@ -16,6 +16,7 @@ import ( ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { ID string diff --git a/core/tasks/parsed/taskinfo.go b/core/tasks/parsed/taskinfo.go index cba7f7bf..d24eec2b 100644 --- a/core/tasks/parsed/taskinfo.go +++ b/core/tasks/parsed/taskinfo.go @@ -1,5 +1,7 @@ package parsed +import "maps" + type TaskInfo interface { TaskID() string Site() string @@ -42,7 +44,11 @@ func (t *Task) DownloadedBytes() int64 { func (t *Task) Processing() map[string]ResourceInfo { t.processingMu.RLock() defer t.processingMu.RUnlock() - return t.processing + // Return a snapshot: callers render it while resource goroutines mutate the + // live map. + snapshot := make(map[string]ResourceInfo, len(t.processing)) + maps.Copy(snapshot, t.processing) + return snapshot } type ResourceInfo interface { diff --git a/core/tasks/telegraph/execute.go b/core/tasks/telegraph/execute.go index d6d891a6..6b7c6402 100644 --- a/core/tasks/telegraph/execute.go +++ b/core/tasks/telegraph/execute.go @@ -66,7 +66,7 @@ func (t *Task) processPic(ctx context.Context, picUrl string, index int) error { return fmt.Errorf("failed to download picture %s: %w", picUrl, err) } defer body.Close() - filename := fmt.Sprintf("%d%s", index+1, path.Ext(picUrl)) + filename := fsutil.SafeBaseName(fmt.Sprintf("%d%s", index+1, path.Ext(picUrl))) if t.cannotStream { cacheFile, err := fsutil.CreateFile(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("tph_%s_%s", t.TaskID(), filename), diff --git a/core/tasks/telegraph/params.go b/core/tasks/telegraph/params.go new file mode 100644 index 00000000..66edcc9b --- /dev/null +++ b/core/tasks/telegraph/params.go @@ -0,0 +1,14 @@ +package telegraph + +import ( + "encoding/json" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + return json.Marshal(taskspec.Telegraph{PhPath: t.PhPath, Pics: t.Pics}) +} diff --git a/core/tasks/telegraph/progress.go b/core/tasks/telegraph/progress.go index 2613e321..2ed4cae6 100644 --- a/core/tasks/telegraph/progress.go +++ b/core/tasks/telegraph/progress.go @@ -44,15 +44,7 @@ func (p *Progress) OnStart(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.ChatID, req) @@ -80,15 +72,7 @@ func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.ChatID, req) diff --git a/core/tasks/telegraph/task.go b/core/tasks/telegraph/task.go index c1b5586a..2a5e9d4d 100644 --- a/core/tasks/telegraph/task.go +++ b/core/tasks/telegraph/task.go @@ -12,6 +12,7 @@ import ( ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { ID string diff --git a/core/tasks/tfile/execute.go b/core/tasks/tfile/execute.go index 8eea959c..2a6fa2b2 100644 --- a/core/tasks/tfile/execute.go +++ b/core/tasks/tfile/execute.go @@ -13,13 +13,15 @@ import ( "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/common/utils/ioutil" "github.com/krau/SaveAny-Bot/config" + "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" "github.com/krau/SaveAny-Bot/pkg/storagetypes" tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" ) -func (t *Task) Execute(ctx context.Context) error { +func (t *Task) Execute(ctx context.Context) (err error) { + defer func() { t.execErr = err }() logger := log.FromContext(ctx).WithPrefix(fmt.Sprintf("file[%s]", t.File.Name())) if t.Progress != nil { t.Progress.OnStart(ctx, t) @@ -29,42 +31,36 @@ func (t *Task) Execute(ctx context.Context) error { } logger.Info("Starting file download") - localFile, err := fsutil.CreateFile(t.localPath) - if err != nil { - return fmt.Errorf("failed to create local file: %w", err) - } - defer func() { - if err := localFile.CloseAndRemove(); err != nil { - logger.Errorf("Failed to close local file: %v", err) - } - }() - wrAt := newWriterAt(ctx, localFile, t.Progress, t) - defer func() { if t.Progress != nil { t.Progress.OnDone(ctx, t, err) } }() - _, err = tdler.NewDownloader(t.File).Parallel(ctx, wrAt) - if err != nil { + if err := tdler.DownloadToCache(ctx, t.File, t.localPath, func(w io.WriterAt) io.WriterAt { + return newWriterAt(ctx, w, t.Progress, t) + }); err != nil { return fmt.Errorf("failed to download file: %w", err) } logger.Infof("File downloaded successfully") - if path.Ext(t.File.Name()) == "" { + if path.Ext(t.File.Name()) == "" && path.Ext(t.Path) == "" { ext := fsutil.DetectFileExt(t.localPath) if ext != "" { t.Path = t.Path + ext } } - var fileStat os.FileInfo - fileStat, err = os.Stat(t.localPath) + fileStat, err := os.Stat(t.localPath) if err != nil { return fmt.Errorf("failed to get file stat: %w", err) } + if t.File.Size() > 0 && fileStat.Size() == 0 { + return fmt.Errorf("downloaded file is empty (expected %d bytes)", t.File.Size()) + } vctx := context.WithValue(ctx, ctxkey.ContentLength, fileStat.Size()) + vctx = storagetypes.WithMessageMetadata(vctx, t.metadataJSON) if caption, ok := sourceCaption(t.File); ok { vctx = storagetypes.WithSourceCaption(vctx, caption) } + savedPath := t.Path err = retry.Retry(func() error { file, err := os.Open(t.localPath) if err != nil { @@ -73,7 +69,7 @@ func (t *Task) Execute(ctx context.Context) error { defer file.Close() uploadProgress, tracksUpload := t.Progress.(UploadProgressTracker) if !tracksUpload { - if err = t.Storage.Save(vctx, file, t.Path); err != nil { + if savedPath, err = storage.SaveWithPath(vctx, t.Storage, file, t.Path); err != nil { return fmt.Errorf("failed to save file: %w", err) } return nil @@ -85,9 +81,10 @@ func (t *Task) Execute(ctx context.Context) error { } if progressSaver, ok := t.Storage.(storage.StorageProgressSaver); ok { err = progressSaver.SaveWithProgress(vctx, file, t.Path, onProgress) + savedPath = t.Path } else { var reader io.Reader = ioutil.NewProgressReader(file, fileStat.Size(), onProgress) - err = t.Storage.Save(vctx, reader, t.Path) + savedPath, err = storage.SaveWithPath(vctx, t.Storage, reader, t.Path) } if err != nil { return fmt.Errorf("failed to save file: %w", err) @@ -97,6 +94,13 @@ func (t *Task) Execute(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to save file after retries: %w", err) } + if err := core.MarkTaskElementDone(ctx, t.ID, t.ElementID); err != nil { + return fmt.Errorf("failed to persist task completion: %w", err) + } + t.saveMetadataSidecar(ctx, savedPath) + if err := os.Remove(t.localPath); err != nil { + logger.Errorf("Failed to remove cache file: %v", err) + } return nil } diff --git a/core/tasks/tfile/params.go b/core/tasks/tfile/params.go new file mode 100644 index 00000000..81bf729a --- /dev/null +++ b/core/tasks/tfile/params.go @@ -0,0 +1,40 @@ +package tfile + +import ( + "encoding/json" + "errors" + + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/enums/ctxkey" + "github.com/krau/SaveAny-Bot/pkg/taskspec" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + messageFile, ok := t.File.(tfilepkg.TGFileMessage) + if !ok || messageFile.Message() == nil { + return nil, errors.New("task has no source message") + } + message := messageFile.Message() + overwrite := false + if t.Ctx != nil { + overwrite, _ = t.Ctx.Value(ctxkey.OverwriteExisting).(bool) + } + return json.Marshal(taskspec.TGFile{ + Overwrite: overwrite, + Done: t.Done, + Items: []taskspec.TGFileItem{{ + ID: t.ElementID, + ChatID: tgutil.ChatIdFromPeer(message.GetPeerID()), + MessageID: message.GetID(), + Name: t.File.Name(), + Storage: t.Storage.Name(), + StorageID: storage.ID(t.Storage), + Path: t.Path, + }}, + }) +} diff --git a/core/tasks/tfile/progress.go b/core/tasks/tfile/progress.go index 240f1dd5..cc9f52f7 100644 --- a/core/tasks/tfile/progress.go +++ b/core/tasks/tfile/progress.go @@ -39,6 +39,7 @@ type Progress struct { lastUpdatePercent atomic.Int32 lastUpdateAt atomic.Int64 updateMu sync.Mutex + edits *tgutil.ProgressEditQueue uploadAttempt int uploadedBytes int64 actualSize int64 @@ -220,9 +221,12 @@ func (p *Progress) editMessage(ctx context.Context, taskID string, message rende } req := buildSingleEditMessageRequest(p.MessageID, taskID, message, cancellable) if ext := tgutil.ExtFromContext(ctx); ext != nil { - if _, err := ext.EditMessage(p.ChatID, req); err != nil { - log.FromContext(ctx).Errorf("Failed to edit file progress message: %v", err) + if p.edits == nil { + p.edits = tgutil.NewProgressEditQueue(tgutil.SendProgressEdit) } + // Queue the edit so a FLOOD_WAIT or a slow RPC never blocks the + // transfer goroutine that is holding the progress mutex. + p.edits.Submit(ctx, p.ChatID, req, !cancellable) } } @@ -233,9 +237,7 @@ func buildSingleEditMessageRequest(messageID int, taskID string, message rendere req.SetEntities(message.Entities) } if cancellable { - req.SetReplyMarkup(&tg.ReplyInlineMarkup{Rows: []tg.KeyboardButtonRow{{ - Buttons: []tg.KeyboardButtonClass{tgutil.BuildCancelButton(taskID)}, - }}}) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(taskID)) } return req } diff --git a/core/tasks/tfile/results.go b/core/tasks/tfile/results.go new file mode 100644 index 00000000..b3150380 --- /dev/null +++ b/core/tasks/tfile/results.go @@ -0,0 +1,40 @@ +package tfile + +import ( + "context" + "errors" + + "github.com/krau/SaveAny-Bot/common/utils/tgutil" + "github.com/krau/SaveAny-Bot/core" + tfilepkg "github.com/krau/SaveAny-Bot/pkg/tfile" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ResultReporter = (*Task)(nil) + +func (t *Task) ItemResults() []core.ItemResult { + result := core.ItemResult{ + Index: 1, + Name: t.File.Name(), + Path: t.Path, + StorageName: t.Storage.Name(), + StorageID: storage.ID(t.Storage), + Size: t.File.Size(), + Status: core.ItemStatusCompleted, + } + if t.execErr != nil { + result.Status = core.ItemStatusFailed + if errors.Is(t.execErr, context.Canceled) { + result.Status = core.ItemStatusCancelled + } + result.Error = t.execErr.Error() + } + if messageFile, ok := t.File.(tfilepkg.TGFileMessage); ok && messageFile.Message() != nil { + msg := messageFile.Message() + result.SourceChatID = tgutil.ChatIdFromPeer(msg.GetPeerID()) + result.SourceMessageID = msg.GetID() + result.MetadataJSON = t.metadataJSON + result.MetadataVersion = t.metadataVersion + } + return []core.ItemResult{result} +} diff --git a/core/tasks/tfile/stream.go b/core/tasks/tfile/stream.go index 881c31e3..960cd9fd 100644 --- a/core/tasks/tfile/stream.go +++ b/core/tasks/tfile/stream.go @@ -7,6 +7,8 @@ import ( "github.com/charmbracelet/log" "github.com/krau/SaveAny-Bot/common/tdler" + "github.com/krau/SaveAny-Bot/pkg/storagetypes" + "github.com/krau/SaveAny-Bot/storage" "golang.org/x/sync/errgroup" ) @@ -16,14 +18,25 @@ func executeStream(ctx context.Context, task *Task) error { pr, pw := io.Pipe() defer pr.Close() errg, uploadCtx := errgroup.WithContext(ctx) + uploadCtx = storagetypes.WithMessageMetadata(uploadCtx, task.metadataJSON) + if caption, ok := sourceCaption(task.File); ok { + uploadCtx = storagetypes.WithSourceCaption(uploadCtx, caption) + } + savedPath := task.Path errg.Go(func() error { - return task.Storage.Save(uploadCtx, pr, task.Path) + var err error + savedPath, err = storage.SaveWithPath(uploadCtx, task.Storage, pr, task.Path) + return err }) wr := newWriter(ctx, pw, task.Progress, task) errg.Go(func() error { defer pw.Close() logger.Info("Starting file download in stream mode") - _, err := tdler.NewDownloader(task.File).Stream(uploadCtx, wr) + downloader, cleanup := tdler.NewDownloader(ctx, task.File) + if cleanup != nil { + defer cleanup(ctx) + } + _, err := downloader.Stream(uploadCtx, wr) if err != nil { logger.Errorf("Failed to download file: %v", err) pw.CloseWithError(err) @@ -40,5 +53,6 @@ func executeStream(ctx context.Context, task *Task) error { return err } logger.Info("File downloaded successfully in stream mode") + task.saveMetadataSidecar(ctx, savedPath) return nil } diff --git a/core/tasks/tfile/tftask.go b/core/tasks/tfile/tftask.go index d2cafa8d..ae1c50bd 100644 --- a/core/tasks/tfile/tftask.go +++ b/core/tasks/tfile/tftask.go @@ -5,24 +5,33 @@ import ( "fmt" "path/filepath" + "github.com/charmbracelet/log" + "github.com/krau/SaveAny-Bot/common/utils/fsutil" "github.com/krau/SaveAny-Bot/config" "github.com/krau/SaveAny-Bot/core" "github.com/krau/SaveAny-Bot/pkg/enums/tasktype" + "github.com/krau/SaveAny-Bot/pkg/metadata" "github.com/krau/SaveAny-Bot/pkg/tfile" "github.com/krau/SaveAny-Bot/storage" ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { - ID string - Ctx context.Context - File tfile.TGFile - Storage storage.Storage - Path string - Progress ProgressTracker - stream bool - localPath string + ID string + ElementID string + Done []string + Ctx context.Context + File tfile.TGFile + Storage storage.Storage + Path string + Progress ProgressTracker + stream bool + localPath string + execErr error + metadataJSON string + metadataVersion int } // Title implements core.Exectable. @@ -42,31 +51,63 @@ func NewTGFileTask( path string, progress ProgressTracker, ) (*Task, error) { + metaJSON, metaVersion := buildFileMetadata(ctx, file) _, ok := stor.(storage.StorageCannotStream) if !config.C().Stream || ok { - cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, file.Name()))) + cachePath, err := filepath.Abs(filepath.Join(config.C().Temp.BasePath, fmt.Sprintf("%s_%s", id, fsutil.SafeBaseName(file.Name())))) if err != nil { return nil, fmt.Errorf("failed to get absolute path for cache: %w", err) } tfile := &Task{ - ID: id, - Ctx: ctx, - File: file, - Storage: stor, - Path: path, - Progress: progress, - localPath: cachePath, + ID: id, + ElementID: id, + Ctx: ctx, + File: file, + Storage: stor, + Path: path, + Progress: progress, + localPath: cachePath, + metadataJSON: metaJSON, + metadataVersion: metaVersion, } return tfile, nil } tfileTask := &Task{ - ID: id, - Ctx: ctx, - File: file, - Storage: stor, - Path: path, - Progress: progress, - stream: true, + ID: id, + ElementID: id, + Ctx: ctx, + File: file, + Storage: stor, + Path: path, + Progress: progress, + stream: true, + metadataJSON: metaJSON, + metadataVersion: metaVersion, } return tfileTask, nil } + +func buildFileMetadata(ctx context.Context, file tfile.TGFile) (string, int) { + if !config.C().Metadata.Enabled { + return "", 0 + } + messageFile, ok := file.(tfile.TGFileMessage) + if !ok || messageFile.Message() == nil { + return "", 0 + } + metaJSON, version, err := metadata.BuildJSON(ctx, messageFile.Message(), file.Name(), file.Size(), config.C().Metadata.IncludeText) + if err != nil { + log.FromContext(ctx).Warnf("Failed to build task metadata: %v", err) + return "", 0 + } + return metaJSON, version +} + +func (t *Task) saveMetadataSidecar(ctx context.Context, actualPath string) { + if !config.C().Metadata.Sidecar || t.metadataJSON == "" || actualPath == "" { + return + } + if err := storage.WriteMetadataSidecar(ctx, t.Storage, actualPath, []byte(t.metadataJSON)); err != nil { + log.FromContext(ctx).Warnf("Failed to save metadata sidecar for %s: %v", actualPath, err) + } +} diff --git a/core/tasks/tfile/writer.go b/core/tasks/tfile/writer.go index 09fc1c59..2b8a1056 100644 --- a/core/tasks/tfile/writer.go +++ b/core/tasks/tfile/writer.go @@ -5,34 +5,21 @@ import ( "io" "sync/atomic" + "github.com/krau/SaveAny-Bot/common/utils/ioutil" "github.com/krau/SaveAny-Bot/pkg/taskevent" ) type ProgressWriterAt struct { ctx context.Context - wrAt io.WriterAt progress ProgressTracker downloaded *atomic.Int64 total int64 info TaskInfo + writer *ioutil.ProgressWriterAt } func (w *ProgressWriterAt) WriteAt(p []byte, off int64) (int, error) { - at, err := w.wrAt.WriteAt(p, off) - if err != nil { - return 0, err - } - downloaded := w.downloaded.Add(int64(at)) - if w.progress != nil { - w.progress.OnProgress(w.ctx, w.info, downloaded, w.total) - } - taskevent.Emit(w.ctx, taskevent.Event{ - TaskID: w.info.TaskID(), - Phase: taskevent.PhaseProgress, - TotalBytes: w.total, - DownloadedBytes: downloaded, - }) - return at, nil + return w.writer.WriteAt(p, off) } func newWriterAt( @@ -41,31 +28,19 @@ func newWriterAt( progress ProgressTracker, taskInfo TaskInfo, ) *ProgressWriterAt { - return &ProgressWriterAt{ + w := &ProgressWriterAt{ ctx: ctx, progress: progress, downloaded: &atomic.Int64{}, total: taskInfo.FileSize(), - wrAt: wrAt, info: taskInfo, } + w.writer = ioutil.NewProgressWriterAt(wrAt, w.record) + return w } -type ProgressWriter struct { - ctx context.Context - wrAt io.Writer - progress ProgressTracker - downloaded *atomic.Int64 - total int64 - info TaskInfo -} - -func (w *ProgressWriter) Write(p []byte) (int, error) { - at, err := w.wrAt.Write(p) - if err != nil { - return 0, err - } - downloaded := w.downloaded.Add(int64(at)) +func (w *ProgressWriterAt) record(n int) { + downloaded := w.downloaded.Add(int64(n)) if w.progress != nil { w.progress.OnProgress(w.ctx, w.info, downloaded, w.total) } @@ -75,7 +50,19 @@ func (w *ProgressWriter) Write(p []byte) (int, error) { TotalBytes: w.total, DownloadedBytes: downloaded, }) - return at, nil +} + +type ProgressWriter struct { + ctx context.Context + progress ProgressTracker + downloaded *atomic.Int64 + total int64 + info TaskInfo + writer *ioutil.ProgressWriter +} + +func (w *ProgressWriter) Write(p []byte) (int, error) { + return w.writer.Write(p) } func newWriter( @@ -84,12 +71,26 @@ func newWriter( progress ProgressTracker, taskInfo TaskInfo, ) *ProgressWriter { - return &ProgressWriter{ + w := &ProgressWriter{ ctx: ctx, progress: progress, downloaded: &atomic.Int64{}, total: taskInfo.FileSize(), - wrAt: wr, info: taskInfo, } + w.writer = ioutil.NewProgressWriter(wr, w.record) + return w +} + +func (w *ProgressWriter) record(n int) { + downloaded := w.downloaded.Add(int64(n)) + if w.progress != nil { + w.progress.OnProgress(w.ctx, w.info, downloaded, w.total) + } + taskevent.Emit(w.ctx, taskevent.Event{ + TaskID: w.info.TaskID(), + Phase: taskevent.PhaseProgress, + TotalBytes: w.total, + DownloadedBytes: downloaded, + }) } diff --git a/core/tasks/transfer/params.go b/core/tasks/transfer/params.go new file mode 100644 index 00000000..7f18e607 --- /dev/null +++ b/core/tasks/transfer/params.go @@ -0,0 +1,30 @@ +package transfer + +import ( + "encoding/json" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + items := make([]taskspec.TransferItem, 0, len(t.elems)) + for i := range t.elems { + elem := &t.elems[i] + items = append(items, taskspec.TransferItem{ + SourceStorage: elem.SourceStorage.Name(), + SourceStorageID: storage.ID(elem.SourceStorage), + SourcePath: elem.SourcePath, + TargetStorage: elem.TargetStorage.Name(), + TargetStorageID: storage.ID(elem.TargetStorage), + TargetPath: elem.TargetPath, + Name: elem.FileInfo.Name, + Size: elem.FileInfo.Size, + ModTime: elem.FileInfo.ModTime, + }) + } + return json.Marshal(taskspec.Transfer{Items: items}) +} diff --git a/core/tasks/transfer/progress.go b/core/tasks/transfer/progress.go index bd538b3e..b96aee7a 100644 --- a/core/tasks/transfer/progress.go +++ b/core/tasks/transfer/progress.go @@ -64,15 +64,7 @@ func (p *Progress) OnStart(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }, - }) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { @@ -143,15 +135,7 @@ func (p *Progress) OnProgress(ctx context.Context, info TaskInfo) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(info.TaskID()), - }, - }, - }, - }) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(info.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { diff --git a/core/tasks/transfer/results.go b/core/tasks/transfer/results.go new file mode 100644 index 00000000..4a0e06fa --- /dev/null +++ b/core/tasks/transfer/results.go @@ -0,0 +1,36 @@ +package transfer + +import ( + "path" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/storage" +) + +var _ core.ResultReporter = (*Task)(nil) + +// ItemResults reports per-file outcomes so a transfer with ignored failures is +// recorded as partial instead of a full success. +func (t *Task) ItemResults() []core.ItemResult { + t.processingMu.RLock() + defer t.processingMu.RUnlock() + results := make([]core.ItemResult, 0, len(t.elems)) + for i := range t.elems { + elem := &t.elems[i] + result := core.ItemResult{ + Index: i + 1, + Name: elem.FileInfo.Name, + Path: path.Join(elem.TargetPath, elem.FileInfo.Name), + StorageName: elem.TargetStorage.Name(), + StorageID: storage.ID(elem.TargetStorage), + Size: elem.FileInfo.Size, + Status: core.ItemStatusCompleted, + } + if err := t.failed[elem.ID]; err != nil { + result.Status = core.ItemStatusFailed + result.Error = err.Error() + } + results = append(results, result) + } + return results +} diff --git a/core/tasks/transfer/task.go b/core/tasks/transfer/task.go index 85e2bdef..728aeb57 100644 --- a/core/tasks/transfer/task.go +++ b/core/tasks/transfer/task.go @@ -14,6 +14,7 @@ import ( ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type TaskElement struct { ID string diff --git a/core/tasks/transfer/taskinfo.go b/core/tasks/transfer/taskinfo.go index cf5a3546..1cc79a17 100644 --- a/core/tasks/transfer/taskinfo.go +++ b/core/tasks/transfer/taskinfo.go @@ -70,3 +70,17 @@ func (t *Task) FailedFiles() []string { } return result } + +func (t *Task) StorageName() string { + if len(t.elems) == 0 { + return "" + } + return t.elems[0].TargetStorage.Name() +} + +func (t *Task) StoragePath() string { + if len(t.elems) == 0 { + return "" + } + return t.elems[0].TargetPath +} diff --git a/core/tasks/ytdlp/execute.go b/core/tasks/ytdlp/execute.go index d554f569..484f0c8d 100644 --- a/core/tasks/ytdlp/execute.go +++ b/core/tasks/ytdlp/execute.go @@ -87,6 +87,9 @@ func (t *Task) Execute(ctx context.Context) error { // task. The bot owns the output directory: the configured template roots every // output in tempDir, and custom -o/--output templates are re-rooted there. func buildDownloadCommand(cfg config.YtdlpConfig, tempDir string, flags []string) (*ytdlp.Command, []string, error) { + if err := validateCustomFlags(flags); err != nil { + return nil, nil, err + } flags, err := rewriteOutputTemplates(flags, tempDir) if err != nil { return nil, nil, err diff --git a/core/tasks/ytdlp/output.go b/core/tasks/ytdlp/output.go index 05e9d68f..ba07fd88 100644 --- a/core/tasks/ytdlp/output.go +++ b/core/tasks/ytdlp/output.go @@ -14,6 +14,47 @@ import ( // and their position, so repeated templates still override each other the way // yt-dlp expects. A -o/--output without a template is rejected, since yt-dlp // would consume the following argument (a URL or another flag) as the template. +// forbiddenYtdlpFlags can execute programs, load configuration, or read and +// write arbitrary files, so they are never accepted from user input. +var forbiddenYtdlpFlags = map[string]struct{}{ + "--exec": {}, + "--exec-before-download": {}, + "--config-location": {}, + "--config-locations": {}, + "--load-info-json": {}, + "--batch-file": {}, + "--downloader": {}, + "--external-downloader": {}, + "--paths": {}, + "--cache-dir": {}, + "--postprocessor-args": {}, + "--ppa": {}, + "--netrc": {}, + "--netrc-cmd": {}, + "--proxy": {}, + "--cookies": {}, + "--cookies-from-browser": {}, + "--plugin-dirs": {}, + "--update": {}, + "--update-to": {}, +} + +func validateCustomFlags(flags []string) error { + for _, flag := range flags { + name := flag + if before, _, ok := strings.Cut(flag, "="); ok { + name = before + } + if _, forbidden := forbiddenYtdlpFlags[name]; forbidden { + return fmt.Errorf("yt-dlp flag %s is not allowed", name) + } + if name == "-P" || strings.HasPrefix(name, "-P") || name == "-U" { + return fmt.Errorf("yt-dlp flag %s is not allowed", name) + } + } + return nil +} + func rewriteOutputTemplates(flags []string, dir string) ([]string, error) { rewritten := make([]string, 0, len(flags)) for i := 0; i < len(flags); i++ { @@ -93,7 +134,7 @@ func splitTypePrefix(template string) (prefix, rest string) { if !ok { return "", template } - for _, k := range strings.Split(key, "+") { + for k := range strings.SplitSeq(key, "+") { if _, known := ytdlpOutputTypes[k]; !known { return "", template } diff --git a/core/tasks/ytdlp/params.go b/core/tasks/ytdlp/params.go new file mode 100644 index 00000000..5aaf334e --- /dev/null +++ b/core/tasks/ytdlp/params.go @@ -0,0 +1,14 @@ +package ytdlp + +import ( + "encoding/json" + + "github.com/krau/SaveAny-Bot/core" + "github.com/krau/SaveAny-Bot/pkg/taskspec" +) + +var _ core.ParamSource = (*Task)(nil) + +func (t *Task) TaskParams() (json.RawMessage, error) { + return json.Marshal(taskspec.YTDLP{URLs: t.URLs, Flags: t.Flags}) +} diff --git a/core/tasks/ytdlp/progress.go b/core/tasks/ytdlp/progress.go index f2ba6924..c31d71c6 100644 --- a/core/tasks/ytdlp/progress.go +++ b/core/tasks/ytdlp/progress.go @@ -60,15 +60,7 @@ func (p *Progress) OnStart(ctx context.Context, task *Task) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(task.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(task.TaskID())) ext.EditMessage(p.chatID, req) } @@ -102,15 +94,7 @@ func (p *Progress) OnProgress(ctx context.Context, task *Task, status string) { } req.SetMessage(text) req.SetEntities(entities) - req.SetReplyMarkup(&tg.ReplyInlineMarkup{ - Rows: []tg.KeyboardButtonRow{ - { - Buttons: []tg.KeyboardButtonClass{ - tgutil.BuildCancelButton(task.TaskID()), - }, - }, - }}, - ) + req.SetReplyMarkup(tgutil.BuildCancelMarkup(task.TaskID())) ext := tgutil.ExtFromContext(ctx) if ext != nil { ext.EditMessage(p.chatID, req) diff --git a/core/tasks/ytdlp/task.go b/core/tasks/ytdlp/task.go index ef945c04..bbbb9751 100644 --- a/core/tasks/ytdlp/task.go +++ b/core/tasks/ytdlp/task.go @@ -10,6 +10,7 @@ import ( ) var _ core.Executable = (*Task)(nil) +var _ core.DestinationInfo = (*Task)(nil) type Task struct { ID string @@ -30,6 +31,17 @@ func (t *Task) Title() string { return fmt.Sprintf("[%s](%d URLs->%s:%s)", t.Type(), urlCount, t.Storage.Name(), t.StorPath) } +func (t *Task) StorageName() string { + if t.Storage == nil { + return "" + } + return t.Storage.Name() +} + +func (t *Task) StoragePath() string { + return t.StorPath +} + // Type implements core.Executable. func (t *Task) Type() tasktype.TaskType { return tasktype.TaskTypeYtdlp diff --git a/database/bot_worker.go b/database/bot_worker.go new file mode 100644 index 00000000..f56d098c --- /dev/null +++ b/database/bot_worker.go @@ -0,0 +1,31 @@ +package database + +import "context" + +func CreateBotWorker(ctx context.Context, worker *BotWorker) error { + return db.WithContext(ctx).Create(worker).Error +} + +func ListBotWorkers(ctx context.Context) ([]BotWorker, error) { + var workers []BotWorker + if err := db.WithContext(ctx).Order("id ASC").Find(&workers).Error; err != nil { + return nil, err + } + return workers, nil +} + +func GetBotWorker(ctx context.Context, id uint) (*BotWorker, error) { + var worker BotWorker + if err := db.WithContext(ctx).First(&worker, id).Error; err != nil { + return nil, err + } + return &worker, nil +} + +func UpdateBotWorker(ctx context.Context, id uint, fields map[string]any) error { + return db.WithContext(ctx).Model(&BotWorker{}).Where("id = ?", id).Updates(fields).Error +} + +func DeleteBotWorker(ctx context.Context, id uint) error { + return db.WithContext(ctx).Unscoped().Delete(&BotWorker{}, id).Error +} diff --git a/database/chat.go b/database/chat.go index 6e6c6ae2..d8136a2a 100644 --- a/database/chat.go +++ b/database/chat.go @@ -1,6 +1,8 @@ package database -import "context" +import ( + "context" +) func (user *User) WatchChat(ctx context.Context, chat WatchChat) error { if len(user.WatchChats) == 0 { diff --git a/database/db.go b/database/db.go index 3c4300e3..c20b025d 100644 --- a/database/db.go +++ b/database/db.go @@ -8,20 +8,19 @@ import ( "time" "github.com/charmbracelet/log" - "github.com/krau/SaveAny-Bot/config" "gorm.io/gorm" glogger "gorm.io/gorm/logger" ) var db *gorm.DB -func Init(ctx context.Context) { +func Init(ctx context.Context, path string) { logger := log.FromContext(ctx) - if err := os.MkdirAll(filepath.Dir(config.C().DB.Path), 0755); err != nil { - logger.Fatal("Failed to create data directory: ", err) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + logger.Fatalf("Failed to create data directory: %v", err) } var err error - db, err = gorm.Open(GetDialect(config.C().DB.Path), &gorm.Config{ + db, err = gorm.Open(GetDialect(path), &gorm.Config{ Logger: glogger.New(logger, glogger.Config{ Colorful: true, SlowThreshold: time.Second * 5, @@ -32,53 +31,29 @@ func Init(ctx context.Context) { PrepareStmt: true, }) if err != nil { - logger.Fatal("Failed to open database: ", err) + logger.Fatalf("Failed to open database: %v", err) } - logger.Debug("Database connected") - if err := db.AutoMigrate(&User{}, &Dir{}, &Rule{}, &WatchChat{}); err != nil { - logger.Fatal("Database migration failed; if upgrading from an old version, try deleting the database file and retrying", "error", err) + if err := os.Chmod(path, 0o600); err != nil { + logger.Warnf("Failed to restrict database permissions: %v", err) } - if err := syncUsers(ctx); err != nil { - logger.Fatal("Failed to sync users:", err) + logger.Debug("Database connected") + if err := migrate(ctx); err != nil { + logger.Fatalf("Database migration failed: %v", err) } - logger.Debug("Database migrated") logger.Info("Database initialized") } -func syncUsers(ctx context.Context) error { - logger := log.FromContext(ctx) - dbUsers, err := GetAllUsers(ctx) - if err != nil { - return fmt.Errorf("failed to get users: %w", err) +func Close() error { + if db == nil { + return nil } - - dbUserMap := make(map[int64]User) - for _, u := range dbUsers { - dbUserMap[u.ChatID] = u - } - - cfgUserMap := make(map[int64]struct{}) - for _, u := range config.C().Users { - cfgUserMap[u.ID] = struct{}{} - } - - for cfgID := range cfgUserMap { - if _, exists := dbUserMap[cfgID]; !exists { - if err := CreateUser(ctx, cfgID); err != nil { - return fmt.Errorf("failed to create user %d: %w", cfgID, err) - } - logger.Infof("Created user from config: %d", cfgID) - } - } - - for dbID, dbUser := range dbUserMap { - if _, exists := cfgUserMap[dbID]; !exists { - if err := DeleteUser(ctx, &dbUser); err != nil { - return fmt.Errorf("failed to delete user %d: %w", dbID, err) - } - logger.Infof("Deleted user not present in config: %d", dbID) - } + sqlDB, err := db.DB() + if err != nil { + return fmt.Errorf("failed to get sql db: %w", err) } + return sqlDB.Close() +} - return nil +func Transaction(ctx context.Context, fn func(tx *gorm.DB) error) error { + return db.WithContext(ctx).Transaction(fn) } diff --git a/database/dir.go b/database/dir.go index 3ae65dda..fed93248 100644 --- a/database/dir.go +++ b/database/dir.go @@ -2,11 +2,11 @@ package database import "context" -func CreateDirForUser(ctx context.Context, userID uint, storageName, path string) error { +func CreateDirForUser(ctx context.Context, userID, storageID uint, path string) error { dir := Dir{ - UserID: userID, - StorageName: storageName, - Path: path, + UserID: userID, + StorageID: storageID, + Path: path, } return db.WithContext(ctx).Create(&dir).Error } @@ -34,24 +34,26 @@ func GetUserDirsByChatID(ctx context.Context, chatID int64) ([]Dir, error) { return GetUserDirs(ctx, user.ID) } -func GetDirsByUserIDAndStorageName(ctx context.Context, userID uint, storageName string) ([]Dir, error) { +func GetDirsByUserIDAndStorageID(ctx context.Context, userID, storageID uint) ([]Dir, error) { var dirs []Dir - err := db.WithContext(ctx).Where("user_id = ? AND storage_name = ?", userID, storageName).Find(&dirs).Error + err := db.WithContext(ctx).Where("user_id = ? AND storage_id = ?", userID, storageID).Find(&dirs).Error return dirs, err } -func GetDirsByUserChatIDAndStorageName(ctx context.Context, chatID int64, storageName string) ([]Dir, error) { +func GetDirsByUserChatIDAndStorageID(ctx context.Context, chatID int64, storageID uint) ([]Dir, error) { user, err := GetUserByChatID(ctx, chatID) if err != nil { return nil, err } - return GetDirsByUserIDAndStorageName(ctx, user.ID, storageName) + return GetDirsByUserIDAndStorageID(ctx, user.ID, storageID) } -func DeleteDirForUser(ctx context.Context, userID uint, storageName, path string) error { - return db.WithContext(ctx).Unscoped().Where("user_id = ? AND storage_name = ? AND path = ?", userID, storageName, path).Delete(&Dir{}).Error +func DeleteDirForUser(ctx context.Context, userID, storageID uint, path string) error { + return db.WithContext(ctx).Unscoped().Where("user_id = ? AND storage_id = ? AND path = ?", userID, storageID, path).Delete(&Dir{}).Error } -func DeleteDirByID(ctx context.Context, id uint) error { - return db.WithContext(ctx).Unscoped().Delete(&Dir{}, id).Error +func DeleteDirByID(ctx context.Context, id uint, userID uint) error { + return db.WithContext(ctx).Unscoped(). + Where("id = ? AND user_id = ?", id, userID). + Delete(&Dir{}).Error } diff --git a/database/driver.go b/database/driver.go index d06744eb..f38e1f7b 100644 --- a/database/driver.go +++ b/database/driver.go @@ -8,5 +8,5 @@ import ( ) func GetDialect(dsn string) gorm.Dialector { - return gormlite.Open(dsn) + return gormlite.Open(dsnWithPragmas(dsn)) } diff --git a/database/driver_glebarez.go b/database/driver_glebarez.go index 7e38b8ea..6ca9805a 100644 --- a/database/driver_glebarez.go +++ b/database/driver_glebarez.go @@ -8,5 +8,5 @@ import ( ) func GetDialect(dsn string) gorm.Dialector { - return sqlite.Open(dsn) + return sqlite.Open(dsnWithPragmas(dsn)) } diff --git a/database/dsn.go b/database/dsn.go new file mode 100644 index 00000000..ac62c79b --- /dev/null +++ b/database/dsn.go @@ -0,0 +1,25 @@ +package database + +import ( + "net/url" + "path/filepath" + "strings" +) + +func dsnWithPragmas(dsn string) string { + if strings.Contains(dsn, "_pragma=") { + return dsn + } + if dsn == ":memory:" || strings.HasPrefix(dsn, "file:") { + return dsn + } + absolute, err := filepath.Abs(dsn) + if err != nil { + return dsn + } + values := url.Values{} + values.Add("_pragma", "foreign_keys(1)") + values.Add("_pragma", "journal_mode(WAL)") + values.Add("_pragma", "busy_timeout(5000)") + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(absolute), RawQuery: values.Encode()}).String() +} diff --git a/database/migration.go b/database/migration.go new file mode 100644 index 00000000..7bb99b2a --- /dev/null +++ b/database/migration.go @@ -0,0 +1,73 @@ +package database + +import ( + "context" + "fmt" + "time" + + "gorm.io/gorm" +) + +// CurrentSchemaVersion is the database schema expected by this build. +// Bump it together with a new entry in schemaMigrations. +const CurrentSchemaVersion = 1 + +type schemaMigration struct { + version int + up func(tx *gorm.DB) error +} + +var schemaMigrations = []schemaMigration{ + {version: 1, up: baselineSchema}, +} + +func baselineSchema(tx *gorm.DB) error { + return tx.AutoMigrate( + &User{}, + &Dir{}, + &Rule{}, + &WatchChat{}, + &Setting{}, + &StorageRecord{}, + &TaskRecord{}, + &TaskItemRecord{}, + &WebSession{}, + &BotWorker{}, + &TaskItemMetadata{}, + ) +} + +func migrate(ctx context.Context) error { + if err := db.WithContext(ctx).Exec(` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at DATETIME NOT NULL +)`).Error; err != nil { + return fmt.Errorf("create schema_migrations: %w", err) + } + + var applied int + if err := db.WithContext(ctx).Raw("SELECT COALESCE(MAX(version), 0) FROM schema_migrations").Scan(&applied).Error; err != nil { + return fmt.Errorf("read schema version: %w", err) + } + if applied > CurrentSchemaVersion { + return fmt.Errorf("database schema version %d is newer than supported version %d", applied, CurrentSchemaVersion) + } + for _, migration := range schemaMigrations { + if migration.version <= applied { + continue + } + if err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := migration.up(tx); err != nil { + return err + } + return tx.Exec( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)", + migration.version, time.Now().UTC(), + ).Error + }); err != nil { + return fmt.Errorf("apply schema migration %d: %w", migration.version, err) + } + } + return nil +} diff --git a/database/model.go b/database/model.go index f3e68ead..77c86453 100644 --- a/database/model.go +++ b/database/model.go @@ -1,44 +1,150 @@ package database import ( + "time" + "gorm.io/gorm" + + "github.com/krau/SaveAny-Bot/pkg/enums/storagefilter" ) type User struct { gorm.Model - ChatID int64 `gorm:"uniqueIndex;not null"` - Silent bool - DefaultStorage string - DefaultDir uint // Dir.ID - Dirs []Dir - ApplyRule bool - Rules []Rule - WatchChats []WatchChat - WatchNotify bool - FilenameStrategy string - FilenameTemplate string - ConflictStrategy string + ChatID int64 `gorm:"uniqueIndex;not null"` + Silent bool + DefaultStorageID uint // StorageRecord.ID + DefaultDirID uint // Dir.ID + Dirs []Dir + ApplyRule bool + Rules []Rule + WatchChats []WatchChat + WatchNotify bool + FilenameStrategy string + FilenameTemplate string + ConflictStrategy string + Enabled bool `gorm:"not null;default:true"` + StorageFilterIDsJSON string `gorm:"column:storage_filter_ids;type:text"` // JSON array of storage IDs + StorageFilterMode storagefilter.Mode `gorm:"size:16;not null;default:blacklist"` // blacklist or whitelist } type WatchChat struct { gorm.Model - UserID uint // User's database ID (not chat ID) - ChatID int64 - Filter string + UserID uint `gorm:"uniqueIndex:idx_watch_user_chat"` + ChatID int64 `gorm:"uniqueIndex:idx_watch_user_chat"` + MessageFilter string // filter type and expression, for example msgre:foo } type Dir struct { gorm.Model - UserID uint - StorageName string - Path string + UserID uint `gorm:"uniqueIndex:idx_dir_user_storage_path"` + StorageID uint `gorm:"uniqueIndex:idx_dir_user_storage_path"` // StorageRecord.ID + Path string `gorm:"uniqueIndex:idx_dir_user_storage_path"` } type Rule struct { gorm.Model - UserID uint - Type string - Data string - StorageName string - DirPath string + UserID uint + Type string + Content string // rule content interpreted according to Type + StorageID uint // StorageRecord.ID + DirPath string +} + +type Setting struct { + Key string `gorm:"primaryKey;size:64"` + Value string `gorm:"type:text;not null"` + UpdatedAt time.Time +} + +type StorageRecord struct { + gorm.Model + Name string `gorm:"uniqueIndex;size:64;not null"` + Type string `gorm:"size:32;not null"` + Enable bool `gorm:"not null"` + ConfigJSON string `gorm:"column:config;type:text;not null"` // JSON storage configuration + ConfigVersion int `gorm:"not null;default:1"` +} + +func (StorageRecord) TableName() string { return "storages" } + +// BotWorker is an optional helper bot used for Telegram storage uploads. +type BotWorker struct { + gorm.Model + Name string `gorm:"uniqueIndex;size:64;not null"` + Token string `gorm:"type:text;not null"` + SessionPath string `gorm:"size:255"` + Enabled bool `gorm:"not null"` +} + +type TaskRecord struct { + ID uint `gorm:"primaryKey"` + TaskID string `gorm:"uniqueIndex;size:64;not null"` + Type string `gorm:"size:32;not null"` + Title string + Status string `gorm:"size:16;index;not null"` + StorageName string + StorageID uint + Path string + Source string `gorm:"size:16;not null"` + Error string `gorm:"type:text"` + Params string `gorm:"type:text"` + ParamsVersion int `gorm:"not null;default:1"` + CreatedAt time.Time + StartedAt *time.Time + FinishedAt *time.Time + UpdatedAt time.Time +} + +// TaskItemRecord is the persisted outcome of one file inside a task. It keeps +// failed items addressable after the task leaves memory, so they can be +// reported and retried. +type TaskItemRecord struct { + ID uint `gorm:"primaryKey"` + TaskID string `gorm:"index:idx_task_item_task_index,unique;size:64;not null"` + Index int `gorm:"index:idx_task_item_task_index,unique;not null"` + Name string + Path string + StorageName string + StorageID uint + Status string `gorm:"size:16;index;not null"` + Error string `gorm:"type:text"` + Size int64 + SourceChatID int64 + SourceMsgID int +} + +func (TaskItemRecord) TableName() string { + return "task_items" +} + +type WebSession struct { + ID string `gorm:"primaryKey;size:64"` + CSRF string `gorm:"size:64;not null"` + CreatedAt time.Time + ExpiresAt time.Time `gorm:"index"` +} + +type TaskFilter struct { + Status string + Type string + Limit int + Offset int +} + +// TaskItemMetadata stores the normalized Telegram message metadata for one +// saved file. It is separate from TaskItemRecord so task lists do not load +// potentially large JSON payloads. +type TaskItemMetadata struct { + ID uint `gorm:"primaryKey"` + TaskID string `gorm:"uniqueIndex:idx_task_metadata_item;size:64;not null"` + ItemIndex int `gorm:"uniqueIndex:idx_task_metadata_item;not null"` + Version int `gorm:"not null;default:1"` + SourceChatID int64 `gorm:"index"` + SourceMessageID int `gorm:"index"` + JSON string `gorm:"column:metadata;type:text;not null"` + CreatedAt time.Time +} + +func (TaskItemMetadata) TableName() string { + return "task_item_metadata" } diff --git a/database/rule.go b/database/rule.go index 937e4abb..7dc5355b 100644 --- a/database/rule.go +++ b/database/rule.go @@ -6,14 +6,22 @@ func CreateRule(ctx context.Context, rule *Rule) error { return db.WithContext(ctx).Create(rule).Error } -func DeleteRule(ctx context.Context, ruleID uint) error { - return db.WithContext(ctx).Unscoped().Delete(&Rule{}, ruleID).Error +func DeleteRule(ctx context.Context, ruleID uint, userID uint) error { + return db.WithContext(ctx).Unscoped(). + Where("id = ? AND user_id = ?", ruleID, userID). + Delete(&Rule{}).Error } func UpdateUserApplyRule(ctx context.Context, chatID int64, applyRule bool) error { return db.WithContext(ctx).Model(&User{}).Where("chat_id = ?", chatID).Update("apply_rule", applyRule).Error } +func GetRulesByUserID(ctx context.Context, userID uint) ([]Rule, error) { + var rules []Rule + err := db.WithContext(ctx).Where("user_id = ?", userID).Find(&rules).Error + return rules, err +} + func GetRulesByUserChatID(ctx context.Context, chatID int64) ([]Rule, error) { var rules []Rule err := db.WithContext(ctx).Where("user_id = (SELECT id FROM users WHERE chat_id = ?)", chatID).Find(&rules).Error diff --git a/database/setting.go b/database/setting.go new file mode 100644 index 00000000..762d28cb --- /dev/null +++ b/database/setting.go @@ -0,0 +1,53 @@ +package database + +import ( + "context" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func GetSetting(ctx context.Context, key string) (string, bool, error) { + var setting Setting + err := db.WithContext(ctx).Where("key = ?", key).First(&setting).Error + if err != nil { + if err == gorm.ErrRecordNotFound { + return "", false, nil + } + return "", false, err + } + return setting.Value, true, nil +} + +func SetSetting(ctx context.Context, key, value string) error { + return db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "key"}}, + DoUpdates: clause.AssignmentColumns([]string{"value", "updated_at"}), + }).Create(&Setting{Key: key, Value: value}).Error +} + +func SetSettingsTx(ctx context.Context, values map[string]string) error { + return Transaction(ctx, func(tx *gorm.DB) error { + for key, value := range values { + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "key"}}, + DoUpdates: clause.AssignmentColumns([]string{"value", "updated_at"}), + }).Create(&Setting{Key: key, Value: value}).Error; err != nil { + return err + } + } + return nil + }) +} + +func ListSettings(ctx context.Context) (map[string]string, error) { + var settings []Setting + if err := db.WithContext(ctx).Find(&settings).Error; err != nil { + return nil, err + } + out := make(map[string]string, len(settings)) + for _, s := range settings { + out[s.Key] = s.Value + } + return out, nil +} diff --git a/database/storage_record.go b/database/storage_record.go new file mode 100644 index 00000000..87807ace --- /dev/null +++ b/database/storage_record.go @@ -0,0 +1,105 @@ +package database + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "gorm.io/gorm" +) + +func CreateStorageRecord(ctx context.Context, record *StorageRecord) error { + return db.WithContext(ctx).Create(record).Error +} + +func ListStorageRecords(ctx context.Context) ([]StorageRecord, error) { + var records []StorageRecord + err := db.WithContext(ctx).Order("name").Find(&records).Error + return records, err +} + +func GetStorageRecord(ctx context.Context, id uint) (*StorageRecord, error) { + var record StorageRecord + if err := db.WithContext(ctx).First(&record, id).Error; err != nil { + return nil, err + } + return &record, nil +} + +func UpdateStorageRecord(ctx context.Context, record *StorageRecord) error { + return db.WithContext(ctx).Save(record).Error +} + +func DeleteStorageRecord(ctx context.Context, id uint) error { + return db.WithContext(ctx).Unscoped().Delete(&StorageRecord{}, id).Error +} + +// MigrateMinioStorageType converts legacy MinIO storage records to S3 and +// normalizes their endpoint to the scheme-based form used by the S3 backend. +func MigrateMinioStorageType(ctx context.Context) error { + var records []StorageRecord + if err := db.WithContext(ctx).Where("type = ?", "minio").Find(&records).Error; err != nil { + return err + } + for _, record := range records { + config := make(map[string]any) + if record.ConfigJSON != "" { + if err := json.Unmarshal([]byte(record.ConfigJSON), &config); err != nil { + return fmt.Errorf("decode legacy MinIO storage %s: %w", record.Name, err) + } + } + if endpoint, ok := config["endpoint"].(string); ok && endpoint != "" && !strings.Contains(endpoint, "://") { + scheme := "http" + if useSSL, _ := config["use_ssl"].(bool); useSSL { + scheme = "https" + } + config["endpoint"] = scheme + "://" + endpoint + } + delete(config, "use_ssl") + raw, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("encode S3 storage %s: %w", record.Name, err) + } + if err := db.WithContext(ctx).Model(&StorageRecord{}). + Where("id = ?", record.ID). + Updates(map[string]any{"type": "s3", "config": string(raw)}).Error; err != nil { + return err + } + } + return nil +} + +// StorageReferences returns how many entities still reference the storage ID. +func StorageReferences(ctx context.Context, storageID uint) (users, dirs, rules int64, err error) { + var allUsers []User + if err = db.WithContext(ctx).Find(&allUsers).Error; err != nil { + return 0, 0, 0, err + } + for _, u := range allUsers { + if u.DefaultStorageID == storageID { + users++ + continue + } + var ids []uint + if u.StorageFilterIDsJSON != "" { + if err := json.Unmarshal([]byte(u.StorageFilterIDsJSON), &ids); err != nil { + return 0, 0, 0, fmt.Errorf("decode storage filter for user %d: %w", u.ChatID, err) + } + } + if slices.Contains(ids, storageID) { + users++ + } + } + if err = db.WithContext(ctx).Model(&Dir{}).Where("storage_id = ?", storageID).Count(&dirs).Error; err != nil { + return + } + if err = db.WithContext(ctx).Model(&Rule{}).Where("storage_id = ?", storageID).Count(&rules).Error; err != nil { + return + } + return users, dirs, rules, nil +} + +func IsNotFound(err error) bool { return errors.Is(err, gorm.ErrRecordNotFound) } diff --git a/database/task_item_metadata.go b/database/task_item_metadata.go new file mode 100644 index 00000000..df48f767 --- /dev/null +++ b/database/task_item_metadata.go @@ -0,0 +1,53 @@ +package database + +import ( + "context" + + "gorm.io/gorm" +) + +// ReplaceTaskItemMetadata stores normalized metadata for a task, replacing any +// rows left by an earlier execution of the same task. +func ReplaceTaskItemMetadata(ctx context.Context, taskID string, records []TaskItemMetadata) error { + return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&TaskItemMetadata{}).Error; err != nil { + return err + } + if len(records) == 0 { + return nil + } + for i := range records { + records[i].TaskID = taskID + } + return tx.CreateInBatches(records, 100).Error + }) +} + +func ListTaskItemMetadata(ctx context.Context, taskID string) ([]TaskItemMetadata, error) { + var records []TaskItemMetadata + err := db.WithContext(ctx).Where("task_id = ?", taskID).Order("item_index ASC").Find(&records).Error + return records, err +} + +func CountTaskMetadata(ctx context.Context, taskIDs []string) (map[string]int, error) { + counts := make(map[string]int, len(taskIDs)) + if len(taskIDs) == 0 { + return counts, nil + } + var rows []struct { + TaskID string + Count int + } + err := db.WithContext(ctx).Model(&TaskItemMetadata{}). + Select("task_id, COUNT(*) AS count"). + Where("task_id IN ?", taskIDs). + Group("task_id"). + Scan(&rows).Error + if err != nil { + return nil, err + } + for _, row := range rows { + counts[row.TaskID] = row.Count + } + return counts, nil +} diff --git a/database/task_record.go b/database/task_record.go new file mode 100644 index 00000000..f34259e7 --- /dev/null +++ b/database/task_record.go @@ -0,0 +1,171 @@ +package database + +import ( + "context" + + "gorm.io/gorm" + + "github.com/krau/SaveAny-Bot/pkg/enums/taskstatus" +) + +func CreateTaskRecord(ctx context.Context, record *TaskRecord) error { + return db.WithContext(ctx).Create(record).Error +} + +func UpdateTaskRecord(ctx context.Context, taskID string, fields map[string]any) error { + return db.WithContext(ctx).Model(&TaskRecord{}).Where("task_id = ?", taskID).Updates(fields).Error +} + +func GetTaskRecord(ctx context.Context, taskID string) (*TaskRecord, error) { + var record TaskRecord + if err := db.WithContext(ctx).Where("task_id = ?", taskID).First(&record).Error; err != nil { + return nil, err + } + return &record, nil +} + +func ListTaskRecords(ctx context.Context, filter TaskFilter) ([]TaskRecord, int64, error) { + query := db.WithContext(ctx).Model(&TaskRecord{}) + if filter.Status != "" { + query = query.Where("status = ?", filter.Status) + } + if filter.Type != "" { + query = query.Where("type = ?", filter.Type) + } + var total int64 + if err := query.Count(&total).Error; err != nil { + return nil, 0, err + } + limit := filter.Limit + if limit <= 0 || limit > 200 { + limit = 50 + } + var records []TaskRecord + err := query.Order("created_at DESC").Limit(limit).Offset(filter.Offset).Find(&records).Error + return records, total, err +} + +// ListQueuedTaskRecords returns every task that is queued for execution, +// including tasks requeued after an interrupted run, oldest first. +func ListQueuedTaskRecords(ctx context.Context) ([]TaskRecord, error) { + var records []TaskRecord + err := db.WithContext(ctx). + Where("status = ?", taskstatus.Queued.String()). + Order("created_at ASC"). + Find(&records).Error + return records, err +} + +// HasUnfinishedTasks reports whether any task is queued or running. +func HasUnfinishedTasks(ctx context.Context) (bool, error) { + var count int64 + err := db.WithContext(ctx).Model(&TaskRecord{}). + Where("status IN ?", []string{taskstatus.Queued.String(), taskstatus.Running.String()}). + Count(&count).Error + return count > 0, err +} + +func DeleteTaskRecord(ctx context.Context, taskID string) error { + return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&TaskItemMetadata{}).Error; err != nil { + return err + } + if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&TaskItemRecord{}).Error; err != nil { + return err + } + return tx.Unscoped().Where("task_id = ?", taskID).Delete(&TaskRecord{}).Error + }) +} + +// ReplaceTaskItems stores the final outcome of every file in a task, +// replacing any previous rows for the same task. +func ReplaceTaskItems(ctx context.Context, taskID string, items []TaskItemRecord) error { + if len(items) == 0 { + return nil + } + return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Unscoped().Where("task_id = ?", taskID).Delete(&TaskItemRecord{}).Error; err != nil { + return err + } + for i := range items { + items[i].TaskID = taskID + } + return tx.CreateInBatches(items, 100).Error + }) +} + +func CountFailedTaskItems(ctx context.Context, taskIDs []string) (map[string]int, error) { + counts := make(map[string]int, len(taskIDs)) + if len(taskIDs) == 0 { + return counts, nil + } + var rows []struct { + TaskID string + Count int + } + err := db.WithContext(ctx).Model(&TaskItemRecord{}). + Select("task_id, COUNT(*) AS count"). + Where("task_id IN ? AND status = ?", taskIDs, taskstatus.Failed.String()). + Group("task_id"). + Scan(&rows).Error + if err != nil { + return nil, err + } + for _, row := range rows { + counts[row.TaskID] = row.Count + } + return counts, nil +} + +func ListTaskItems(ctx context.Context, taskID string) ([]TaskItemRecord, error) { + var items []TaskItemRecord + if err := db.WithContext(ctx).Where("task_id = ?", taskID).Order("id ASC").Find(&items).Error; err != nil { + return nil, err + } + return items, nil +} + +// ListFailedTaskItems returns failed items for the given task IDs, capped per +// task so a large batch cannot blow up a list response. +func ListFailedTaskItems(ctx context.Context, taskIDs []string, perTask int) (map[string][]TaskItemRecord, error) { + out := make(map[string][]TaskItemRecord, len(taskIDs)) + if len(taskIDs) == 0 { + return out, nil + } + if perTask <= 0 { + perTask = 20 + } + // Cap rows per task in SQL so a task with a large number of failures + // cannot make every list request load all of them. + const query = ` +SELECT * FROM ( + SELECT task_items.*, ROW_NUMBER() OVER (PARTITION BY task_id ORDER BY id) AS row_num + FROM task_items + WHERE task_id IN ? AND status = ? +) WHERE row_num <= ?` + var items []TaskItemRecord + if err := db.WithContext(ctx).Raw(query, taskIDs, taskstatus.Failed.String(), perTask).Scan(&items).Error; err != nil { + return nil, err + } + for _, item := range items { + group := out[item.TaskID] + if len(group) >= perTask { + continue + } + out[item.TaskID] = append(group, item) + } + return out, nil +} + +// MarkInterruptedTaskRecords requeues tasks that were running when the +// process stopped so they can be rebuilt from their persisted parameters. +func MarkInterruptedTaskRecords(ctx context.Context) error { + return db.WithContext(ctx).Model(&TaskRecord{}). + Where("status = ?", taskstatus.Running.String()). + Updates(map[string]any{ + "status": taskstatus.Queued.String(), + "error": "", + "started_at": nil, + "finished_at": nil, + }).Error +} diff --git a/database/user.go b/database/user.go index 7f631b5e..4279403a 100644 --- a/database/user.go +++ b/database/user.go @@ -4,13 +4,15 @@ import ( "context" "gorm.io/gorm/clause" + + "github.com/krau/SaveAny-Bot/pkg/enums/storagefilter" ) func CreateUser(ctx context.Context, chatID int64) error { if _, err := GetUserByChatID(ctx, chatID); err == nil { return nil } - return db.Create(&User{ChatID: chatID}).Error + return db.Create(&User{ChatID: chatID, StorageFilterMode: storagefilter.Blacklist}).Error } func GetAllUsers(ctx context.Context) ([]User, error) { diff --git a/database/websession.go b/database/websession.go new file mode 100644 index 00000000..deca8395 --- /dev/null +++ b/database/websession.go @@ -0,0 +1,30 @@ +package database + +import ( + "context" + "time" +) + +func CreateWebSession(ctx context.Context, session *WebSession) error { + return db.WithContext(ctx).Create(session).Error +} + +func GetWebSession(ctx context.Context, id string) (*WebSession, error) { + var session WebSession + if err := db.WithContext(ctx).Where("id = ? AND expires_at > ?", id, time.Now()).First(&session).Error; err != nil { + return nil, err + } + return &session, nil +} + +func DeleteWebSession(ctx context.Context, id string) error { + return db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&WebSession{}).Error +} + +func DeleteOtherWebSessions(ctx context.Context, keepID string) error { + return db.WithContext(ctx).Unscoped().Where("id <> ?", keepID).Delete(&WebSession{}).Error +} + +func DeleteExpiredWebSessions(ctx context.Context) error { + return db.WithContext(ctx).Unscoped().Where("expires_at <= ?", time.Now()).Delete(&WebSession{}).Error +} diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 5659d1e7..3bcf458d 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -4,12 +4,18 @@ services: container_name: saveany-bot restart: unless-stopped ports: - - "8080:8080" + - "28080:28080" + # Web UI 默认仅监听 127.0.0.1:28080;带访问 Token 的地址会打印在日志中 + # The Web UI binds to 127.0.0.1:28080 by default; the tokenized URL is printed in the log + # 如需对外开放,取消注释并设置 SAVEANY_LISTEN + # To expose it, uncomment SAVEANY_LISTEN + # environment: + # SAVEANY_ADMIN_PASSWORD: change-me + # SAVEANY_LISTEN: 0.0.0.0:28080 volumes: - ./data:/app/data - - ./config.toml:/app/config.toml - ./downloads:/app/downloads - ./cache:/app/cache # 使用 host 模式以便访问宿主机服务 (如代理) # 如果你对 Docker 网络模式熟悉, 可以自行修改 - network_mode: host \ No newline at end of file + network_mode: host diff --git a/docker-compose.yml b/docker-compose.yml index 2e358555..ba7374a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,13 +3,17 @@ services: image: ghcr.io/krau/saveany-bot:latest container_name: saveany-bot restart: unless-stopped - ports: - - "8080:8080" + # Web UI 默认仅监听 127.0.0.1:28080;带访问 Token 的地址会打印在日志中 + # The Web UI binds to 127.0.0.1:28080 by default; the tokenized URL is printed in the log + # 如需对外开放,取消注释并设置 SAVEANY_LISTEN + # To expose it, uncomment SAVEANY_LISTEN + # environment: + # SAVEANY_ADMIN_PASSWORD: change-me + # SAVEANY_LISTEN: 0.0.0.0:28080 volumes: - ./data:/app/data - - ./config.toml:/app/config.toml - ./downloads:/app/downloads - ./cache:/app/cache # 使用 host 模式以便访问宿主机服务 (如代理) # 如果你对 Docker 网络模式熟悉, 可以自行修改 - network_mode: host \ No newline at end of file + network_mode: host diff --git a/docs/content/en/contribute/_index.md b/docs/content/en/contribute/_index.md index e7f84d5a..21240092 100644 --- a/docs/content/en/contribute/_index.md +++ b/docs/content/en/contribute/_index.md @@ -12,6 +12,7 @@ Here are some guidelines and suggestions for contributing code. You don't have t - **Open an issue before adding new features**, so we can discuss design and implementation details and avoid work that doesn't fit the project design. - **Use modern development tools**, format your code before committing, and keep the style consistent. - **Use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/)**, and avoid vague or overly simple commit messages. +- **See [Testing](testing) for the seeded environment and end-to-end suites.** ## Contributing New Storage Backend diff --git a/docs/content/en/contribute/testing.md b/docs/content/en/contribute/testing.md new file mode 100644 index 00000000..cb6deb27 --- /dev/null +++ b/docs/content/en/contribute/testing.md @@ -0,0 +1,80 @@ +--- +title: "Testing" +weight: 10 +--- + +# Testing + +The project ships a seeded local environment and two end-to-end suites that do +not need Telegram credentials. + +## Seeded environment + +```bash +make testenv +``` + +The command creates a disposable data directory and prints: + +- the Web UI URL +- the admin password (`saveany-test` by default) +- a seeded local storage and whitelisted user +- a sample URL that can be used as a direct-links task + +Use `--keep` to retain the generated data directory, or pass `--data-dir` to +reuse an existing one. Telegram is left stopped, so the web console and the +non-Telegram task pipeline are available immediately. + +## Backend end-to-end tests + +```bash +make e2e-api +``` + +The suite builds the binary, starts the seeded environment on an ephemeral +port and drives the public API with a real HTTP client. It covers authentication +and CSRF, onboarding, storage and user lifecycle, settings updates, the SPA +asset fallback, and a direct-links task that downloads from a local HTTP server +into the seeded storage. + +The tests live under `e2e/` behind the `e2e` build tag, so `go test ./...` does +not start the environment. + +## Telegram end-to-end tests + +```bash +make e2e-telegram +``` + +This suite runs the real bot handlers, dispatcher, task queue, and storage +backends against an in-memory `tg.Invoker`. It covers `/start`, group admission, +saving a Telegram document to local storage (including progress edits), and +uploading it to a Telegram storage backend. No Telegram account or network is +used: gotd's client, sender, downloader, and uploader run unchanged against the +fake RPC layer. + +## Web end-to-end tests + +```bash +make e2e-web-setup # once, downloads the Chromium build +make e2e-web +``` + +The suite starts the seeded environment and drives the built web UI with +Playwright. It covers login, storage testing and creation, and creating a +direct-links task that reaches the completed state. Test artifacts are written +to `tmp/e2e-web-results/`. + +Playwright is a dev dependency of `web/ui` and its version is locked by +`pnpm-lock.yaml`. `make e2e-web-setup` installs the matching Chromium build. + +## Combined + +```bash +make e2e +``` + +Runs the API, Telegram, and browser suites after building the web UI. The fake +Telegram layer covers commands, downloads, uploads, and group admission without +an account. Real Telegram server semantics, userbot login, and live rate limits +still need a test account. diff --git a/docs/content/en/deployment/configuration/_index.md b/docs/content/en/deployment/configuration/_index.md index 65387045..9923da53 100644 --- a/docs/content/en/deployment/configuration/_index.md +++ b/docs/content/en/deployment/configuration/_index.md @@ -1,270 +1,106 @@ --- -title: "Configuration Guide" +title: "Configuration" +weight: 11 --- -# Configuration Guide - -SaveAnyBot uses the toml format for its configuration files. You can learn more about toml syntax on the [TOML official website](https://toml.io/). - -SaveAnyBot needs to read a `config.toml` file in the working directory as its configuration file. If this file is missing, a default file will be created, and the bot will attempt to load configuration from environment variables. - -Here is an example of a minimal configuration file: - -```toml -[telegram] -token = "1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZ" - -[[users]] -# telegram user id -id = 777000 -blacklist = true - -[[storages]] -name = "Local Storage" -type = "local" -enable = true -base_path = "./downloads" -``` - -## Detailed Configuration - -### Global Configuration - -- `lang`: The language used by the Bot, default is `zh-CN` (Simplified Chinese). `en` is used for English. -- `stream`: Whether to enable Stream mode, default is `false`. When enabled, the Bot will stream files directly to storage endpoints (if supported), without downloading them locally. -{{< hint warning >}} -Stream mode is very useful for deployment environments with limited disk space, but it also has some drawbacks: -
-
    -
  • Cannot use multi-threading to download files from Telegram, resulting in slower speeds.
  • -
  • Higher task failure rate when the network is unstable.
  • -
  • Cannot process files in the middle layer, such as automatic file type identification.
  • -
  • Not supported by all storage endpoints; unsupported endpoints may downgrade to normal mode or fail to upload.
  • -
-{{< /hint >}} -- `workers`: Number of tasks to process simultaneously, default is 3. -- `threads`: Number of threads used when downloading files, default is 4. Only effective when Stream mode is not enabled. -- `retry`: Number of retries when a task fails, default is 3. -- `proxy`: Global proxy configuration. After setting this, all network connections inside the program will try to use this proxy. Optional. - -```toml -lang = "en" -stream = false -workers = 3 -threads = 4 -retry = 3 -proxy = "socks5://127.0.0.1:7890" -``` - -### Telegram Configuration - -- `token`: Your Telegram Bot Token, which can be obtained by creating a Bot through [BotFather](https://t.me/botfather). -- `app_id`, `app_hash`: Telegram API ID & Hash, obtained by creating an application at [Telegram API](https://my.telegram.org/apps). Default values will be used if not provided. -- `flood_retry`: Number of retries for flood control, default is 5. -- `rpc_retry`: Number of retries for RPC requests, default is 5. -- `proxy`: Proxy configuration, optional. - - `enable`: Whether to enable the proxy. - - `url`: Proxy address -- `userbot`: Userbot configuration, optional. - - `enable`: Enable userbot integration. Requires logging in with a user account; you should use your own API ID & Hash when enabling this. - - `session`: Path to the userbot session file, default is `data/usersession.db`. - -{{< hint warning >}} -After enabling userbot integration, the bot can download files from private channels and groups, but there is an unavoidable risk of the account being banned. -
-On the first start after enabling userbot, you need to input phone number, 2FA and verification code in the terminal. -
-If you deploy with Docker, please run the container with `-it` for an interactive environment, then perform the login. -{{< /hint >}} - -```toml -[telegram] -token = "1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZ" -app_id = 1025907 -app_hash = "452b0359b988148995f22ff0f4229750" -flood_retry = 5 -rpc_retry = 5 -[telegram.proxy] -enable = false -url = "socks5://127.0.0.1:7890" -[telegram.userbot] -enable = false -session = "data/usersession.db" -``` - -### Aria2 Configuration - -Aria2 is a powerful download manager that supports HTTP/HTTPS, FTP, BitTorrent, and other protocols. When enabled, the bot can use the `/aria2dl` command to download files via Aria2. - -- `enable`: Whether to enable Aria2 support, default is `false` -- `url`: Aria2 RPC address, typically `http://localhost:6800/jsonrpc` -- `secret`: Aria2 RPC secret, if you configured `rpc-secret` in Aria2, you need to fill it in here -- `remove_after_transfer`: Whether to remove local files downloaded by Aria2 after transfer, default is `true` - -{{< hint info >}} -Aria2 needs to be installed and running separately. You can refer to the [Aria2 official documentation](https://aria2.github.io/) to learn how to install and configure Aria2. -{{< /hint >}} - -```toml -[aria2] -enable = true -url = "http://localhost:6800/jsonrpc" -secret = "your-rpc-secret" -remove_after_transfer = true -``` - -### yt-dlp Configuration - -Configures the behavior of the `/ytdlp` command and the `ytdlp` HTTP-API task type when no custom flags are passed. - -- `max_height`: Default maximum video resolution by height in pixels (e.g. `1080`, `720`). `0` means no limit (best available). Ignored when `format` is set. -- `format`: A raw yt-dlp format selector (`-f`). When set, it takes precedence over `max_height` and gives you full control, e.g. `bv*[height<=720]+ba/b`. -- `recode`: The target video container yt-dlp recodes into after download (e.g. `mp4`). Leave empty to disable recoding. -- `filename_template`: yt-dlp output template for downloaded file names, default `%(title)s.%(ext)s`. Directories in the template are created below the bot's download directory and kept in the storage path. -- `restrict_filenames`: Restrict file names to ASCII characters (yt-dlp `--restrict-filenames`), which also drops spaces and `&`. Disabled by default. - -{{< hint info >}} -`max_height`, `format` and `recode` only apply when using the `/ytdlp` command (or the `ytdlp` API task type) without passing any custom flags. `filename_template` and `restrict_filenames` also apply when other flags are passed, unless you pass the corresponding flag yourself (`-o/--output`, `--restrict-filenames` or `--no-restrict-filenames`). -{{< /hint >}} - -```toml -[ytdlp] -max_height = 1080 -format = "" # e.g. "bv*[height<=720]+ba/b" -recode = "mp4" # empty disables recoding -filename_template = "%(title)s.%(ext)s" # e.g. "%(uploader)s - %(title)s.%(ext)s" -restrict_filenames = false -``` - -### HTTP API Configuration - -When enabled, SaveAny-Bot exposes an HTTP API for creating/querying/canceling tasks programmatically. See [HTTP API](../../usage/api) for the full endpoint reference. - -- `enable`: Whether to enable the HTTP API server, default is `false`. -- `host`: Bind address, default `0.0.0.0`. -- `port`: Listen port, default `8080`. -- `token`: Authentication token. **Strongly recommended** — if empty, the API is exposed without any authentication. - -```toml -[api] -enable = false -host = "0.0.0.0" -port = 8080 -token = "your-token" -``` - -### Log Configuration - -- `level`: Log level. One of `debug`, `info`, `warn`, `error`, `fatal`. Default is `info`. - -```toml -[log] -level = "info" -``` - -### Storage Endpoints List - -The storage endpoints list is used to define the storage locations supported by the Bot. Each storage endpoint needs to specify a name, type, and related configuration, using the double bracket syntax `[[storages]]`. - -Each storage endpoint requires at least the following fields: - -- `name`: Storage endpoint name, used for identification in the Bot, must be unique. -- `enable`: Whether to enable this storage endpoint, default is `true`. -- `type`: Storage endpoint type, currently supports the following types: - - `local`: Local disk - - `alist`: Alist - - `webdav`: WebDAV - - `s3`: aws S3 and other S3 compatible services - - `rclone`: Uses rclone to implement uploads - - `telegram`: Upload to Telegram - -Example, this is a configuration that includes local storage and webdav storage: - -```toml -[[storages]] -name = "Local Storage" -type = "local" -enable = true -# Custom configuration for local type storage -base_path = "./downloads" - -[[storages]] -name = "WebDAV" -type = "webdav" -enable = true -# Custom configuration for webdav type storage -url = "https://example.com/webdav" -base_path = "/path/to/webdav" -username = "your_username" -password = "your_password" -``` - -For custom configuration items for all storage endpoints, see [Storage Configuration](./storages) - -### User List - -The user list is used to define access control for storage endpoints. Each user needs to specify a Telegram User ID, defined using the double bracket syntax `[[users]]`. - -- `id`: The user's Telegram User ID -- `storages`: Filtered list of storage endpoints, defined by storage endpoint names, default is whitelist mode (i.e., only allows access to storage endpoints in the list) -- `blacklist`: Whether to enable blacklist mode, default is `false`. If blacklist mode is enabled, the user is allowed to access only storage endpoints that are **not** in the list. - -Example, this is a configuration containing three users: user `123123` can only access local storage, user `456456` can only access storage other than WebDAV, and user `789789` has blacklist mode enabled but no storage endpoints specified, so they can access all storage: +# Configuration -```toml -[[users]] -id = 123123 -storages = ["Local Storage"] - -[[users]] -id = 456456 -storages = ["WebDAV"] -blacklist = true - -[[users]] -id = 789789 -storages = [] -blacklist = true -``` +Settings are stored in the SQLite database under the data directory (default `data/saveany.db`) and managed from the web console. The first start uses defaults and prints a direct web URL with an access token; set `SAVEANY_ADMIN_PASSWORD` to choose the fallback password. -### Events - -Event hooks allow you to run custom commands based on task status while the bot is processing tasks. Currently only arbitrary command execution is supported, configured via `[hook.exec]`. +## Configuration pages -Supported event types: +| Page | Contents | +|---|---| +| Guide | First-run checklist and everyday usage | +| Telegram | Bot token, API ID/hash, proxy, Userbot login | +| Storages | Local disk, WebDAV, Alist, S3, Rclone, Telegram, and more | +| Users | Telegram user IDs, default storage, directories, rules, and storage access | +| Automation | Per-user directories, rules, watched chats, and peer sync | +| Settings | Task concurrency, language, proxy, media type filter, Aria2, yt-dlp, parsers, cache, logging, web, and API | +| Parsers | Install JavaScript parser plugins | +| System | Runtime information, restart, version checks, and updates | -- `task_before_start`: Before a task starts -- `task_success`: After a task completes successfully -- `task_fail`: After a task fails -- `task_cancel`: After a task is cancelled +Some settings require a process restart, including task concurrency, global proxy, database session path, cache settings, plugins, hooks, and the web listen address. The page marks fields that need a restart. -The configured value must be a full shell command line. The bot will execute this command when the event occurs. Example: +### Automation -```toml -[hook.exec] -task_before_start = "echo 'task is about to start'" -task_success = "bash /path/to/success_script.sh" -task_fail = "curl -X POST https://example.com/api/notify -d 'task failed'" -task_cancel = "bash /path/to/cancel_script.sh" -``` +The Automation page manages a selected user's directories, rules, watched chats, and Userbot peer sync. Directories can be marked as the default destination; rules can be enabled, disabled, added, deleted, or imported from presets. ### Parsers -Parsers give the bot the ability to handle non-Telegram files, such as downloading files from other websites. Configure them via `[parsers]`. +The Parsers page installs JavaScript parser plugins when parser plugins are enabled in Settings. Upload a `.js` plugin file or paste its code; it becomes available without a restart. -```toml -[parsers] -plugin_enable = true # Whether to enable parser plugins -plugin_dirs = ["./plugins"] # Plugin directories, can be multiple -``` +### Telegram -The above settings only control JavaScript-based parser plugins. The bot also has built-in parsers implemented in Go, which are enabled by default. +Create a bot with [@BotFather](https://t.me/BotFather). `API ID` and `API Hash` have defaults and usually need no changes; use your own [Telegram API](https://my.telegram.org/apps) credentials when enabling the Userbot. -### Miscellaneous +Saving bot settings starts the connection automatically. Userbot login happens on the Telegram page, with no terminal interaction inside the container. -```toml -no_clean_cache = false # Whether not to clear the cache folder when exiting -# Temporary download folder configuration -[temp] -base_path = "./cache" -``` \ No newline at end of file +### Helper bots + +Helper bots are optional. Add them in the **Helper bots** section of the **Telegram** page with a BotFather token, then add each bot to the destination Telegram chat with permission to send files. Telegram storage uploads are distributed across running helpers that can access the destination; helpers without access are skipped and uploads fall back to the primary bot. Helper bots do not handle commands. + +To split large file downloads across helpers, set a private **Relay channel ID** on the **Telegram** page, make sure the primary bot can post there, and add at least two helper bots to that channel. The primary bot forwards incoming files to the relay channel, and each helper obtains its own file reference before downloading assigned chunks. This requires non-stream mode and files at or above the configurable **Relay minimum file size** (8 MB by default), and falls back to the single-client downloader when the relay or helper pool is unavailable. Forwarded relay messages are kept by default; disable **Keep forwarded channel messages** to delete them after download. + +Transfer concurrency can be tuned under **Settings → Core**: + +| Setting | Default | Meaning | +|---|---|---| +| `Download threads per file` | 4 | Base threads used by automatic scheduling | +| `Download thread override` | 0 | 0 uses automatic sizing; a positive value is the exact total thread count | +| `Automatic thread cap` | 16 | Maximum threads when automatic sizing is used | +| `Global transfer budget` | 0 | 0 uses `workers × threads`; downloads and uploads share this cap | +| `Relay minimum file size (MB)` | 8 | Smaller files skip helper relays | +| `Chunk retry count` | 20 | Retries for timeouts or connection failures on one chunk | + +### Media filter + +Use **Settings → Media filter** to restrict the bot to selected Telegram media types. Enter one value per line under **Allowed media types**; leave the field empty to allow everything: + +| Value | Matches | +|---|---| +| `image` | Telegram photos, image documents, stickers, and animated images | +| `video` | Telegram video documents, including video messages | +| `audio` | Telegram audio and voice messages | +| `document` | Other document files | + +Filtering happens before download tasks are created. Non-matching media is not downloaded and does not appear in the task list. Media groups keep only matching items; a group with no matching items is skipped silently. This is an instance-wide setting and takes effect immediately. + +The filter currently covers Telegram media entry points: direct and forwarded media, media groups, batch `/save`, message links, watched chats, and Telegraph images. Parser results, direct HTTP links, and yt-dlp output are not filtered. Storage rules only change the destination path; they do not skip files. + +### Message metadata + +Enable **Settings → Message metadata** to store normalized Telegram message data for new Telegram file tasks. The local database keeps source chat, message ID, date, sender, text or caption, reply/forward references, album grouping, and media attributes. The task details API and Web console can export the stored JSON. + +**Store message text** controls whether the message body and extracted tags are kept. Access hashes, tokens, raw Telegram objects, and old task data are not stored. Metadata applies only to files saved after the setting is enabled. + +**Write sidecar JSON** writes `.meta.json` next to each saved file. It is off by default. Telegram storage does not upload the sidecar; it can instead consume the metadata to restore captions or media groups using the Telegram storage options. + +### Storages + +Add and enable storages on the **Storages** page. Local disk needs only a base path: + +- In Docker, usually `/app/downloads`. +- For a binary deployment, use `./downloads` or an absolute path. + +See [Storage configuration](./storages) for backend fields. + +### Users + +Only enabled Telegram user IDs on the **Users** page can use the bot. A new user needs at least the numeric user ID; the default storage can be set later. + +## Environment variables + +| Variable | Purpose | +|---|---| +| `SAVEANY_DATA_DIR` | Data directory, default `./data` | +| `SAVEANY_DB_PATH` | SQLite file path, default `/saveany.db` | +| `SAVEANY_LISTEN` | Web listen address, default `127.0.0.1:28080`; use `0.0.0.0:28080` to expose it | +| `SAVEANY_ADMIN_PASSWORD` | Set the fallback admin password on first start; otherwise one is generated and logged | + +## Security + +- The web console binds to `127.0.0.1:28080` by default. Expose it publicly only through a reverse proxy with HTTPS. +- The `Access token` setting enables direct Web UI access through `/?token=...` and Bearer authentication for `/api/v1`. When empty, a token is generated on every start and printed in the log. +- Change the generated fallback admin password immediately after the first password login. +- Never expose `data/`, bot tokens, Userbot sessions, or access tokens. diff --git a/docs/content/en/deployment/configuration/storages.md b/docs/content/en/deployment/configuration/storages.md index 44d7780f..4771fad5 100644 --- a/docs/content/en/deployment/configuration/storages.md +++ b/docs/content/en/deployment/configuration/storages.md @@ -4,7 +4,9 @@ title: "Storage Configuration" # Storage Configuration -Please first read the [Configuration Guide](../) to understand the basic format of the configuration file. +Add storages on the **Storages** page in the web console and fill in the fields below. + +Storage names can contain spaces, Unicode, and symbols (up to 64 characters, excluding control characters). Quote names containing spaces when using bot commands such as `/dirs` or `/transfer`. Rename a storage on the same page at any time; user defaults, directories, rules, and storage filters reference the stable storage ID, so they are unaffected by the name change. ## Alist @@ -12,7 +14,7 @@ Please first read the [Configuration Guide](../) to understand the basic format Stream mode is not supported. -```toml +```text url = "https://alist.example.com" # URL of Alist username = "your_username" # Username for Alist password = "your_password" # Password for Alist @@ -27,14 +29,14 @@ token = "your_token" `type=local` -```toml +```text base_path = "./downloads" # Base path for local storage, all files will be stored under this path ``` ## WebDAV `type=webdav` -```toml +```text url = "https://webdav.example.com" # URL of WebDAV username = "your_username" # Username for WebDAV password = "your_password" # Password for WebDAV @@ -45,8 +47,8 @@ base_path = "/path/to/webdav" # Base path in WebDAV, all files will be stored un `type=s3` -```toml -endpoint = "s3.example.com" # Endpoint for S3, defaults to AWS S3 endpoint if not set +```text +endpoint = "https://s3.example.com" # Endpoint for S3, defaults to AWS S3 endpoint if not set region = "us-east-1" # Region for S3 access_key_id = "your_access_key_id" # Access key ID for S3 secret_access_key = "your_secret_access_key" # Secret access key for S3 @@ -75,21 +77,25 @@ If you are using a third-party S3-compatible service, it usually uses path-style Stream mode is not supported. -```toml -chat_id = "123456789" # Telegram chat ID, the bot will send files to this chat +```text +chat_id = "-1001234567890" # Bot API chat ID (positive for users, negative for basic groups, -100... for channels/supergroups) force_file = false # Force sending as file, default is false skip_large = false # Skip large files, default is false. If enabled, files exceeding Telegram's limit will not be uploaded. split_large_video = false # Losslessly split oversized videos into one album of up to 10 independently playable parts. Falls back to ZIP parts on failure or when more than 10 parts are required. split_size_mb = 0 # Split size in MB. 0 uses the uploader account limit: 2000 MB for bots/regular users and 4000 MB for Premium users. Oversized non-video files use ZIP parts. Ignored when skip_large is true. +use_metadata_caption = false # Use metadata text as caption when the live source caption is absent +use_metadata_album = false # Use metadata grouped_id to preserve source media groups when live grouping is absent ``` +When message metadata is enabled, Telegram storage consumes it directly instead of uploading a `.meta.json` file. `use_metadata_caption` restores the original message text as the caption when the live source caption is unavailable; `use_metadata_album` uses metadata `grouped_id` to send the files as a media group when live grouping is unavailable. Both options are off by default. For albums, the caption is attached only to the first item, matching Telegram client behavior. + ## Rclone `type=rclone` Supports multiple cloud storage services through the [rclone](https://rclone.org/) command-line tool. You need to install rclone and configure remote storage first. -```toml +```text # Remote name configured in rclone, can be any remote defined in rclone.conf remote = "mydrive" # Base path in the remote storage, all files will be stored under this path @@ -117,7 +123,7 @@ rclone supports many cloud storage services, including but not limited to: After configuring Google Drive, you can configure the storage like this: -```toml +```text [[storages]] name = "GoogleDrive" type = "rclone" @@ -128,7 +134,7 @@ base_path = "/SaveAnyBot" If using a custom rclone config file: -```toml +```text [[storages]] name = "MyRemote" type = "rclone" diff --git a/docs/content/en/deployment/installation.md b/docs/content/en/deployment/installation.md index 89e986f7..f40dc357 100644 --- a/docs/content/en/deployment/installation.md +++ b/docs/content/en/deployment/installation.md @@ -1,30 +1,55 @@ --- -title: "Installation and Updates" +title: "Installation and updates" +weight: 10 --- -# Installation and Updates +# Installation and updates -## Deploy from Pre-compiled Binary (Recommended) +Start with an empty data directory, then configure the bot, storage, and users in the web console. -Download the binary file for your platform from the [Release](https://github.com/krau/SaveAny-Bot/releases) page. +## Binary deployment -Create a `config.toml` file in the extracted directory, refer to the [Configuration Guide](../configuration) to edit the configuration file. - -Run: +Download and extract a release for your platform: ```bash chmod +x saveany-bot ./saveany-bot ``` -### Daemon +The default data directory is `data/` under the current working directory. The web UI listens on `127.0.0.1:28080` by default. At startup it prints a direct URL with an access token: + +```text +Web UI available at http://127.0.0.1:28080/?token=... +``` + +Open that URL; the token creates an authenticated session. Unless an access token is set under **Settings → Web**, a new token is generated on every start. + +To expose the web UI on another interface or port, pass the listen address on the command line: + +```bash +./saveany-bot --listen 0.0.0.0:28080 +``` + +The equivalent environment variable is `SAVEANY_LISTEN`: + +```bash +SAVEANY_LISTEN='0.0.0.0:28080' ./saveany-bot +``` + +The generated admin password remains available for password login; set `SAVEANY_ADMIN_PASSWORD` to choose it. It is printed only on the first start. After signing in, change it from the sidebar **Change password** dialog. Open the logged URL and follow the **Guide** page: + +1. Enter the bot token on the Telegram page. +2. Add at least one storage on the Storages page. +3. Add your Telegram user ID on the Users page. + +### Process supervision {{< tabs "daemon" >}} -{{< tab "systemd (Regular Linux)" >}} +{{< tab "systemd (Linux)" >}} -Create a file /etc/systemd/system/saveany-bot.service and write the following content: +Create `/etc/systemd/system/saveany-bot.service`: -{{< codeblock >}} +```ini [Unit] Description=SaveAnyBot After=systemd-user-sessions.service @@ -37,135 +62,97 @@ Restart=always [Install] WantedBy=multi-user.target -{{< /codeblock >}} - -Enable startup on boot and start the service: +``` -{{< codeblock >}} +```bash systemctl enable --now saveany-bot -{{< /codeblock >}} +``` {{< /tab >}} - {{< tab "procd (OpenWrt)" >}} -

Add Boot Autostart Service

- -Create a file /etc/init.d/saveanybot, refer to wrt_init and modify as needed: +Create `/etc/init.d/saveanybot`, based on [wrt_init](https://github.com/krau/SaveAny-Bot/blob/main/docs/confs/wrt_init), and adjust the working directory and executable path: -{{< codeblock >}} +```sh #!/bin/sh /etc/rc.common - -#This is the OpenWRT init.d script for SaveAnyBot - -START=99 +START=99 STOP=10 description="SaveAnyBot" - WORKING_DIR="/mnt/mmc1-1/SaveAnyBot" EXEC_PATH="$WORKING_DIR/saveany-bot" -start() { - echo "Starting SaveAnyBot..." - cd $WORKING_DIR - $EXEC_PATH & -} -stop() { - echo "Stopping SaveAnyBot..." - killall saveany-bot -} -reload() { - stop - start -} - -{{< /codeblock >}} - -Set permissions: - -{{< codeblock >}} -chmod +x /etc/init.d/saveanybot -{{< /codeblock >}} - -Then copy the file to /etc/rc.d and rename it to S99saveanybot, also set permissions: - -{{< codeblock >}} -chmod +x /etc/rc.d/S99saveanybot -{{< /codeblock >}} - -

Add Shortcut Commands

- -Create a file /usr/bin/sabot, refer to wrt_bin and modify as needed. Note that the file encoding here only supports ANSI 936. - -Then set permissions: - -{{< codeblock >}} -chmod +x /usr/bin/sabot -{{< /codeblock >}} +start() { cd $WORKING_DIR && $EXEC_PATH & } +stop() { killall saveany-bot; } +``` -Usage: sudo sabot start|stop|restart|status|enable|disable +```bash +chmod +x /etc/init.d/saveanybot +ln -s /etc/init.d/saveanybot /etc/rc.d/S99saveanybot +``` {{< /tab >}} {{< /tabs >}} - -## Deploy Using Docker +## Docker deployment ### Docker Compose -Download the [docker-compose.yml](https://github.com/krau/SaveAny-Bot/blob/main/docker-compose.yml) file, create a new `config.toml` file in the same directory, refer to [config.example.toml](https://github.com/krau/SaveAny-Bot/blob/main/config.example.toml) to edit the configuration file. - -Start: +Download [docker-compose.yml](https://github.com/krau/SaveAny-Bot/blob/main/docker-compose.yml), adjust ports and paths if needed, then start: ```bash docker compose up -d +docker compose logs -f saveany-bot +``` + +Default mounts: + +- `./data:/app/data` — settings, task history, and sessions +- `./downloads:/app/downloads` — default local storage +- `./cache:/app/cache` — download cache + +Set an admin password, or read the generated one from the log. Set the listen address for bridge networking: + +```yaml +environment: + SAVEANY_ADMIN_PASSWORD: change-me + SAVEANY_LISTEN: 0.0.0.0:28080 ``` ### Docker -```shell +```bash docker run -d --name saveany-bot \ - -v /path/to/config.toml:/app/config.toml \ - -v /path/to/downloads:/app/downloads \ + -p 28080:28080 \ + -e SAVEANY_LISTEN=0.0.0.0:28080 \ + -v ./data:/app/data \ + -v ./downloads:/app/downloads \ + -v ./cache:/app/cache \ ghcr.io/krau/saveany-bot:latest ``` -{{< hint info >}} -About Docker image variants -
-
    -
  • Default: Includes all features and dependencies, larger in size. Use this if you don't have special requirements.
  • -
  • micro: Slimmed-down image with some optional dependencies removed, smaller in size.
  • -
  • pico: Minimal image containing only core features, smallest in size.
  • -
-You can pull different variants by specifying tags, for example: ghcr.io/krau/saveany-bot:micro -
-For more details about the variants, see the Dockerfile in the project root. -{{< /hint >}} +Read the tokenized URL from the logs and open it. For local storage in Docker, use `/app/downloads` as the base path. ## Updates -If you deployed from pre-compiled binaries, use the following CLI command to update: +### Binary ```bash ./saveany-bot up ``` -(`upgrade` is also available as an alias.) - -The command replaces the binary in place; a running bot keeps the old version until it is restarted. You can also send `/update` to the bot: it downloads the new version, replaces its own binary and restarts itself in place (on Windows it exits instead, and the service manager starts the new version). +You can also send `/update` in Telegram. The updated process restarts itself; on Windows, restart it from your service manager. -If you deployed with Docker, use the following commands to update: - -docker: +### Docker ```bash docker pull ghcr.io/krau/saveany-bot:latest docker restart saveany-bot ``` -docker compose: +### Docker Compose ```bash docker compose pull -docker compose restart -``` \ No newline at end of file +docker compose up -d +``` + +Back up `data/` before updating and keep the Docker volume. diff --git a/docs/content/en/deployment/migration.md b/docs/content/en/deployment/migration.md new file mode 100644 index 00000000..b4d20563 --- /dev/null +++ b/docs/content/en/deployment/migration.md @@ -0,0 +1,132 @@ +--- +title: "Migrate from v0" +weight: 15 +--- + +# Migrate from v0 + +v1 stores configuration in a local SQLite database. On first startup, it imports a legacy v0 `config.toml` once, then all settings are managed from the web console. + +v1 does not migrate Telegram sessions. The v0 `session.db` and `usersession.db` files were created by gotgproto and are no longer read. Migration imports the legacy configuration; v1 uses a new database. The bot signs in again with the configured token, and an enabled Userbot must sign in again from the web console. Back up all existing files before upgrading. + +## 1. Stop and back up + +Stop v0 and back up the configuration, data, and downloads: + +```bash +docker compose down +cp -a config.toml data downloads /path/to/backup/ +``` + +For a binary deployment, stop the process and back up the same directories. + +Do not run v0 and v1 at the same time, and never share one `data/` directory between two instances. + +## 2. Keep the data directory + +v1 does not reuse the v0 database. Keep the old file as a backup and let v1 create a new `data/saveany.db`: + +| v0 file | v1 default | Purpose | +|---|---|---| +| `data/saveany.db` | not reused | Keep as a backup; v1 creates a new database | + +Do not point v1 at the v0 database with `--db-path` or `SAVEANY_DB_PATH`; the v0 schema is not migrated. Move or rename the old `data/saveany.db` out of the data directory before the first v1 start; v1 must create a new database at the default path. Keep the old file as a backup. + +The v0 `data/session.db` and `data/usersession.db` are ignored; delete them after confirming they are backed up. v1 uses new session files: + +| File | Purpose | +|---|---| +| `data/bot_session.db` | Bot session | +| `data/userbot_session.db` | Userbot session | + +The bot signs in again with the configured token. If the Userbot is enabled, sign in again on the Telegram page later. + +## 3. Import the configuration on first startup + +v1 must be able to read the legacy `config.toml` during its first startup. In Docker, mount it only for that run: + +```yaml +services: + saveany-bot: + image: ghcr.io/krau/saveany-bot:latest + restart: unless-stopped + ports: + - "28080:28080" + environment: + SAVEANY_LISTEN: 0.0.0.0:28080 + volumes: + - ./data:/app/data + - ./config.toml:/app/config.toml:ro + - ./downloads:/app/downloads + - ./cache:/app/cache +``` + +```bash +docker compose up -d +docker compose logs -f saveany-bot +``` + +For a binary deployment: + +```bash +./saveany-bot --config /path/to/config.toml +``` + +Without `--config`, v1 checks `./config.toml` and then `/etc/saveany/config.toml`. + +After the import log appears, remove the `config.toml` mount and restart. v1 writes an import marker to the database and will not import it again. The legacy `db.session` and `telegram.userbot.session` values are ignored; v1 always uses the new session file paths. + +## 4. Sign in and verify + +The startup log prints a direct Web UI URL with an access token; open it to sign in. If `SAVEANY_ADMIN_PASSWORD` was not set on the first run, the fallback admin password is printed in the log as well: + +```bash +docker compose logs saveany-bot | grep -E "Web UI available at|Generated web admin password" +``` + +Open **Guide** after signing in and verify: + +- The bot has reconnected with the current token. +- If the Userbot is enabled, it has signed in again and shows its account. +- At least one storage is enabled and has no initialization error. +- The Users list contains your old accounts. Old users that were not present in the v0 config are disabled; enable them manually if needed. +- Storages, users, and settings imported from `config.toml` are present. +- Recreate directories, rules, and watched chats in v1; v0 database rows are not migrated. + +Then send a file or Telegram message link to the bot and confirm a task completes. + +## 5. After migration + +- Later changes to `config.toml` are **ignored**. Use the web console. +- MinIO storage entries are converted to `s3` on startup; review their endpoint, credentials, and bucket in the web console. +- Back up the entire `data/` directory before updates and keep the Docker volume. +- Update Docker with `docker compose pull && docker compose up -d`. +- If you change the Bot token, delete `data/bot_session.db` before restarting to avoid an old session mismatch. + +## Troubleshooting + +**Nothing was imported** + +Check that the config path is correct and the file is valid TOML. The import marker is written only after a successful import. Existing storage names and settings groups are not overwritten. + +**A legacy session was not migrated** + +v1 does not read the old gotgproto `session.db` or `usersession.db`. The bot signs in again with the current token; if the Userbot is enabled, sign in again on the Telegram page. Keep the old files in your backup and delete them when they are no longer needed. + +**The old database is not migrated** + +v1 does not reuse the v0 database or its `[db] path`. Keep the old file as a backup and let v1 create a new database. + +**Local storage paths stopped working** + +v0 resolved local storage paths relative to the process working directory. In Docker this is usually `/app`; make sure `downloads/` is still mounted at the same path. + +**The admin password is unknown** + +Find `Generated web admin password` in the first startup log. Only a hash is stored, so the password cannot be recovered from the database. If the log is gone, back up the database, delete the `auth` setting, and restart to generate a new password. + +## Related + +- [Installation and updates](../installation) +- [Web console configuration](../configuration/) +- [Usage](../../usage/) diff --git a/docs/content/en/usage/_index.md b/docs/content/en/usage/_index.md index 0a0932e5..62a39f32 100644 --- a/docs/content/en/usage/_index.md +++ b/docs/content/en/usage/_index.md @@ -7,10 +7,38 @@ weight: 10 This page introduces some of Save Any Bot's features and basic usage. If you can't find what you need here, please also see the [Configuration Guide](../deployment/configuration) or ask in GitHub [Discussions](https://github.com/krau/SaveAny-Bot/discussions). +## Bot Commands + +| Command | Purpose | +|---|---| +| `/start` | Open help and the command list | +| `/save` | Save a replied media message, or batch-save a message range | +| `/download` | Download links using Direct, Aria2, or Video | +| `/transfer` | Transfer files between storages | +| `/tasks` | Inspect and manage running, queued, and failed tasks | +| `/config` | User settings: filename and conflict strategy, silent mode, default storage | +| `/template` | Set the custom filename template | +| `/dirs` | Manage storage directories | +| `/rules` | Manage storage rules | +| `/watch` | Watch chats and manage notifications/peer sync (UserBot) | +| `/parser` | Manage parser plugins | +| `/update` | Check for updates | + + ## File Transfer To use the bot's Telegram file saving feature, you need to send or forward the following types of messages to the bot: 1. File or media messages, such as images, videos, documents, etc. 2. Telegram message links, for example: `https://t.me/acherkrau/1097`. **Even if the channel prohibits forwarding and saving, the bot can still download its files.** -3. Telegra.ph article links. The bot will download all images in the article. \ No newline at end of file +3. Telegra.ph article links. The bot will download all images in the article. + +## Group Chats + +The bot works in basic groups and supergroups. Broadcast channels are not supported. + +- In groups the bot only reacts to explicit commands and button callbacks; ordinary media and link messages are not saved automatically. +- To save a file from the group, reply to it with `/save`, or use `/save@YourBot` in groups. Other commands such as `/download` work as usual. +- Only whitelisted users can use the bot. Messages from other members are ignored silently, and their button clicks receive a permission alert. +- Messages sent by anonymous administrators in supergroups have no identifiable user and are ignored. +- With the default Privacy Mode, the bot only receives commands addressed to it, mentions, and replies to its own messages. To let it receive all group messages, use `/setprivacy` in [@BotFather](https://t.me/BotFather) to disable Privacy Mode and re-add the bot to the group. Even then, the bot only reacts to commands and buttons in groups. \ No newline at end of file diff --git a/docs/content/en/usage/api.md b/docs/content/en/usage/api.md index 740158f3..936f7e13 100644 --- a/docs/content/en/usage/api.md +++ b/docs/content/en/usage/api.md @@ -9,41 +9,29 @@ SaveAny-Bot provides an HTTP API that allows you to programmatically create down ## Enabling the API -Add or modify the following section in `config.toml`: - -```toml -[api] -enable = true -host = "0.0.0.0" # Bind address, default 0.0.0.0 -port = 8080 # Listen port, default 8080 -token = "your-token" # Auth token — strongly recommended -``` - -You can also override these settings with environment variables (prefix `SAVEANY_`): +The API shares the web console listener and port; there is nothing separate to enable. The web console listens on `127.0.0.1:28080` by default. Set an access token under **Settings → Web**: -| Environment Variable | Config Key | -|---|---| -| `SAVEANY_API_ENABLE` | `api.enable` | -| `SAVEANY_API_HOST` | `api.host` | -| `SAVEANY_API_PORT` | `api.port` | -| `SAVEANY_API_TOKEN` | `api.token` | +- `Access token`: used for direct Web UI access through `/?token=...` and with `Authorization: Bearer ` for the API. +- `Web listen address`: shared by the console and API; use `0.0.0.0:28080` when it must be reachable from other hosts. -{{< hint warning >}} -If `token` is empty, the API server will be accessible **without any authentication**, which is a security risk. -{{< /hint >}} +All endpoints live under `/api/v1`. ## Authentication -When `token` is configured, all API requests must include a Bearer token in the HTTP header: +Set an access token under **Settings → Web**, then send it in the request header: ``` Authorization: Bearer ``` -On authentication failure, the server returns `401`: +If no access token is configured, a random token is generated on every start and printed in the log as part of the direct Web UI URL. That value also works as a Bearer token for the current process. + +You can also use the web console login cookie. Non-GET requests made with a cookie also require the `X-CSRF-Token` header. + +Authentication failures return `401`: ```json -{ "error": "unauthorized", "message": "invalid token" } +{ "error": "unauthorized", "message": "authentication required" } ``` ## Error Response Format @@ -62,7 +50,6 @@ Common error codes: | Error Code | HTTP Status | Meaning | |---|---|---| | `unauthorized` | 401 | Authentication failed | -| `method_not_allowed` | 405 | Wrong HTTP method | | `invalid_request` | 400 | Malformed request body or parameters | | `task_creation_failed` | 400 | Failed to create task | | `task_not_found` | 404 | Task ID does not exist | @@ -80,7 +67,7 @@ No authentication required. **Response `200 OK`:** ```json -{ "status": "ok" } +{ "status": "ok", "version": "1.0.0" } ``` --- @@ -94,14 +81,44 @@ Returns all currently loaded storage backends. ```json { "storages": [ - { "name": "local", "type": "local" }, - { "name": "MyMinio", "type": "s3" } + { + "storage_id": 1, + "name": "local", + "type": "local", + "enable": true, + "config": { "base_path": "/app/downloads" }, + "config_version": 1 + } ] } ``` --- +### GET /api/v1/meta — API and Capability Metadata + +Returns the frozen v1 capability vocabulary and the database schema version. + +```json +{ + "api_version": "v1", + "version": "1.0.0", + "schema_version": 1, + "chat_id_format": "bot_api", + "task_types": ["tgfiles", "tphpics", "parseditem", "directlinks", "aria2", "ytdlp", "transfer"], + "task_statuses": ["queued", "running", "completed", "partial", "failed", "cancelled"], + "task_item_statuses": ["completed", "failed", "skipped", "cancelled"], + "storage_types": ["local", "webdav", "alist", "telegram", "s3", "rclone"], + "rule_types": ["FILENAME-REGEX", "MESSAGE-REGEX", "IS-ALBUM"], + "filename_strategies": ["default", "message", "template"], + "conflict_strategies": ["rename", "ask", "overwrite", "skip"] +} +``` + +Chat IDs use the Bot API form: positive IDs for users, negative IDs for basic groups, and `-100...` IDs for supergroups and channels. + +--- + ### GET /api/v1/task-types — List Supported Task Types **Response `200 OK`:** @@ -109,13 +126,8 @@ Returns all currently loaded storage backends. ```json { "types": [ - "directlinks", - "ytdlp", - "aria2", - "parseditem", - "tgfiles", - "tphpics", - "transfer" + { "type": "directlinks", "label": "directlinks" }, + { "type": "tgfiles", "label": "tgfiles" } ] } ``` @@ -129,7 +141,7 @@ Inspect a Telegram message link, direct media URL, or other yt-dlp-supported med Request: ```bash -curl -X GET "http://localhost:8080/api/v1/media-metadata?url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dxxx" \ +curl -X GET "http://127.0.0.1:28080/api/v1/media-metadata?url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dxxx" \ -H "Authorization: Bearer YOUR_TOKEN" ``` @@ -163,7 +175,7 @@ Authorization: Bearer ```json { "type": "", - "storage": "", + "storage_id": 1, "path": "", "webhook": "", "params": { } @@ -173,7 +185,7 @@ Authorization: Bearer | Field | Type | Required | Description | |---|---|---|---| | `type` | string | Yes | Task type — see below | -| `storage` | string | Yes | Target storage name, must match a name in your config | +| `storage_id` | uint | Yes | Target storage ID | | `path` | string | No | Subdirectory path within the storage | | `webhook` | string | No | Callback URL invoked when the task reaches a terminal state | | `params` | object | Yes | Type-specific parameters — see below | @@ -182,9 +194,11 @@ Authorization: Bearer ```json { - "task_id": "abc123xyz", - "type": "directlinks", - "status": "queued", + "task_id": "abc123xyz", + "type": "directlinks", + "status": "queued", + "storage_id": 1, + "storage_name": "local", "created_at": "2026-03-11T10:00:00Z" } ``` @@ -198,7 +212,7 @@ Download one or more files from direct HTTP/HTTPS URLs. ```json { "type": "directlinks", - "storage": "local", + "storage_id": 1, "path": "downloads", "params": { "urls": [ @@ -224,7 +238,7 @@ Download videos or audio via yt-dlp, supporting YouTube, Bilibili, and 1000+ oth ```json { "type": "ytdlp", - "storage": "local", + "storage_id": 1, "path": "videos", "params": { "urls": ["https://www.youtube.com/watch?v=xxx"], @@ -241,7 +255,7 @@ Download videos or audio via yt-dlp, supporting YouTube, Bilibili, and 1000+ oth ##### aria2 — Aria2 Download {{< hint warning >}} -Requires Aria2 to be enabled and configured (RPC) in the config file. +Requires Aria2 to be enabled and configured (RPC) under **Settings → Aria2**. {{< /hint >}} Download files via the Aria2 download manager, supporting HTTP/HTTPS, FTP, BitTorrent (magnet links, torrent files), and more. @@ -249,7 +263,7 @@ Download files via the Aria2 download manager, supporting HTTP/HTTPS, FTP, BitTo ```json { "type": "aria2", - "storage": "local", + "storage_id": 1, "path": "downloads", "params": { "urls": ["magnet:?xt=urn:btih:..."], @@ -270,7 +284,7 @@ Hand a URL off to a registered JS plugin or built-in parser for processing and d ```json { "type": "parseditem", - "storage": "local", + "storage_id": 1, "path": "parsed", "params": { "url": "https://some-site.com/page" @@ -298,7 +312,7 @@ If the message is part of a media group (album), all files in the group are down ```json { "type": "tgfiles", - "storage": "local", + "storage_id": 1, "path": "telegram", "params": { "message_links": [ @@ -311,7 +325,10 @@ If the message is part of a media group (album), all files in the group are down | params field | Type | Required | Description | |---|---|---|---| -| `message_links` | []string | Yes | List of Telegram message links, at least 1 | +| `message_links` | []string | No | List of Telegram message links; required unless `chat_id` and `message_range` are used | +| `chat_id` | string | No | Bot API chat ID or username used with `message_range` | +| `message_range` | string | No | Inclusive message range, for example `100-120` | +| `filter` | string | No | Optional regular expression matching message text or file names | ##### tphpics — Telegraph Article Images @@ -322,7 +339,7 @@ Supported URL prefixes: `https://telegra.ph/`, `http://telegra.ph/`, `https://te ```json { "type": "tphpics", - "storage": "local", + "storage_id": 1, "path": "telegraph", "params": { "telegraph_url": "https://telegra.ph/Some-Article-01-01" @@ -339,34 +356,34 @@ Supported URL prefixes: `https://telegra.ph/`, `http://telegra.ph/`, `https://te Transfer files directly between two storage backends without going through Telegram. The source storage must support both listing and reading. {{< hint info >}} -For `transfer` tasks, the top-level `storage` field is still required for validation, but the actual storages used are determined by `source_storage` and `target_storage` inside `params`. +For `transfer` tasks, the top-level `storage_id` field is still required for validation, but the actual storages used are determined by `source_storage_id` and `target_storage_id` inside `params`. {{< /hint >}} ```json { "type": "transfer", - "storage": "local", + "storage_id": 1, "params": { - "source_storage": "MyS3", - "source_path": "backups/", - "target_storage": "LocalDisk", - "target_path": "restored/" + "source_storage_id": 2, + "source_path": "backups/", + "target_storage_id": 1, + "target_path": "restored/" } } ``` | params field | Type | Required | Description | |---|---|---|---| -| `source_storage` | string | Yes | Source storage name | +| `source_storage_id` | uint | Yes | Source storage ID | | `source_path` | string | Yes | Path within the source storage; must contain at least one file | -| `target_storage` | string | Yes | Target storage name | +| `target_storage_id` | uint | Yes | Target storage ID | | `target_path` | string | Yes | Destination path within the target storage | --- ### GET /api/v1/tasks — List All Tasks -Returns all tasks created via the API, newest first (by creation time). Task records are stored in memory only and are cleared on restart. Tasks that have reached a terminal state are also evicted 24 hours after their last update; the sweep runs every 10 minutes. +Returns tasks newest first (by creation time). Task records and their creation parameters are persisted in SQLite; queued and interrupted tasks are resumed after a restart. Terminal records remain until deleted through the API. **Response `200 OK`:** @@ -378,7 +395,8 @@ Returns all tasks created via the API, newest first (by creation time). Task rec "type": "directlinks", "status": "running", "title": "file.zip", - "storage": "local", + "storage_id": 1, + "storage_name": "local", "path": "downloads", "error": "", "created_at": "2026-03-11T10:00:00Z", @@ -411,25 +429,113 @@ The `progress` field is included when either `total_bytes` or `total_files` is g - `400 invalid_request` — no task ID in path - `404 task_not_found` — task does not exist +Task objects include `metadata_available` when at least one item has stored message metadata. + --- -### DELETE /api/v1/tasks/{task_id} — Cancel Task +### GET /api/v1/tasks/{task_id}/metadata — Get Message Metadata -**Path parameter:** `task_id` +Returns the normalized Telegram message metadata recorded for every saved file in a Telegram file task. Enable **Settings → Message metadata** before creating the task. Metadata is always stored in the local database. The optional **Write sidecar JSON** setting also writes `.meta.json` beside the saved file; Telegram storage skips sidecars. + +**Path parameter:** `task_id` — the task whose metadata should be returned. **Response `200 OK`:** ```json -{ "message": "task cancelled successfully" } +{ + "task_id": "abc123xyz", + "items": [ + { + "index": 1, + "name": "saved-video.mp4", + "path": "downloads/saved-video.mp4", + "metadata": { + "version": 1, + "message": { + "id": 123, + "chat": { "id": -1001234567890, "title": "Channel" }, + "sender": { "id": 123456789, "name": "Alice" }, + "text": "caption", + "grouped_id": 999 + }, + "file": { + "name": "saved-video.mp4", + "media_type": "document", + "mime_type": "video/mp4", + "size": 12345678, + "width": 1920, + "height": 1080, + "duration": 123.4 + } + } + } + ] +} ``` +Message text is omitted when **Store message text** is disabled. Access hashes, tokens, and raw Telegram objects are never stored. + **Error responses:** - `400 invalid_request` — no task ID in path - `404 task_not_found` — task does not exist +- `404 metadata_not_found` — the task has no recorded metadata + +--- + +### POST /api/v1/tasks/{task_id}/retry — Retry Failed Files + +Creates a new task from the failed files of a terminal task. The bot fetches the original messages again, which also refreshes their file references. Supported for Telegram file tasks (`tgfiles`). + +**Path parameter:** `task_id` — the task to retry. + +**Response `202 Accepted`:** + +```json +{ "task_id": "d4f7u1crvimcs4n4j8e1", "status": "queued", "file_count": 2 } +``` + +**Error responses:** +- `409 task_active` — the task is still queued or running +- `409 nothing_to_retry` — the task has no failed or unfinished files +- `409 storage_missing` / `409 source_unavailable` +- `400 retry_unsupported` — the task type cannot be retried +- `503 client_unavailable` — no Telegram client is running + +Task responses include per-file outcomes: list responses carry the total in `failed_count` plus up to five failed items per task in `failed_items`, and the single-task response carries every recorded file in `items`. + +--- + +### POST /api/v1/tasks/{task_id}/cancel — Cancel Task + +Cancels a queued or running task. + +**Path parameter:** `task_id` — the task to cancel. + +**Response `202 Accepted`:** + +```json +{ "task_id": "abc123xyz", "status": "cancelled" } +``` + +**Error responses:** +- `409 task_not_active` — the task is already terminal +- `404 task_not_found` — task does not exist - `500 cancel_failed` — cancellation failed --- +### DELETE /api/v1/tasks/{task_id} — Delete Task + +Deletes a terminal task and its recorded items. Use `POST /api/v1/tasks/{task_id}/cancel` to cancel a queued or running task. + +**Path parameter:** `task_id` + +**Response `204 No Content`:** +- `409 task_active` — cancel the task before deleting it +- `404 task_not_found` — task does not exist + +--- + ## Task Statuses | Status | Meaning | @@ -437,17 +543,20 @@ The `progress` field is included when either `total_bytes` or `total_files` is g | `queued` | Task is queued and waiting to run | | `running` | Task is currently executing | | `completed` | Task finished successfully | +| `partial` | Task finished, but one or more files failed | | `failed` | Task encountered an error | -| `cancelled` | Task was cancelled via the DELETE endpoint | +| `cancelled` | Task was cancelled through the cancel endpoint | + +Queued tasks and tasks interrupted while running keep their creation parameters and are re-enqueued after a restart. Telegram downloads resume from the partial cache; files whose upload already completed are skipped. --- ## Webhook Callbacks -When a `webhook` URL is provided in the create request, SaveAny-Bot sends a `POST` request to that URL once the task reaches `completed` or `failed`. +When a `webhook` URL is provided in the create request, SaveAny-Bot sends a `POST` request to that URL once the task reaches `completed`, `partial`, or `failed`. {{< hint info >}} -Cancelled tasks do **not** trigger a webhook — cancellation is already known to whoever issued the DELETE. Each task fires at most one callback. +Cancelled tasks do **not** trigger a webhook — cancellation is already known to whoever requested it. Each task fires at most one callback. {{< /hint >}} The callback is sent asynchronously, so the task is not delayed by a slow or unreachable endpoint. @@ -466,13 +575,14 @@ User-Agent: SaveAny-Bot/1.0 "task_id": "abc123xyz", "type": "directlinks", "status": "completed", - "storage": "local", + "storage_id": 1, + "storage_name": "local", "path": "downloads", "completed_at": "2026-03-11T10:01:00Z", "error": "" } ``` -`completed_at` is only present when status is `completed` or `failed`. `error` is only present when non-empty. +`completed_at` is only present when status is `completed`, `partial`, or `failed`. `error` is only present when non-empty. **Retry policy:** Up to 3 attempts. A non-2xx response or a transport error triggers a retry after 100ms, then 400ms. Each request has a 30-second timeout. diff --git a/docs/content/en/usage/aria2.md b/docs/content/en/usage/aria2.md index eead799b..21e8f90b 100644 --- a/docs/content/en/usage/aria2.md +++ b/docs/content/en/usage/aria2.md @@ -1,41 +1,37 @@ --- -title: "Aria2 Download" +title: "Aria2 Downloads" weight: 6 --- -# Aria2 Download +# Aria2 Downloads {{< hint warning >}} -This feature requires enabling Aria2 in the configuration file and configuring the RPC connection. +This feature requires enabling Aria2 under **Settings → Aria2** and configuring the RPC connection. {{< /hint >}} -Use the `/aria2dl` command to download files via the Aria2 download manager, supporting HTTP/HTTPS, FTP, BitTorrent, and other protocols. +Aria2 downloads are available through `/download`. Send links, then choose **Aria2**. It supports HTTP/HTTPS, FTP, BitTorrent, and magnet links. ```bash -/aria2dl [uri2] [uri3] ... +/download [uri2] [uri3] ... ``` Examples: ```bash -# Download HTTP link -/aria2dl https://example.com/file.zip +# Download an HTTP link +/download https://example.com/file.zip -# Download magnet link -/aria2dl magnet:?xt=urn:btih:... +# Download a magnet link +/download magnet:?xt=urn:btih:... -# Download torrent file (need to upload .torrent file first) -/aria2dl https://example.com/file.torrent +# Download a torrent file +/download https://example.com/file.torrent ``` -Configure Aria2: +Choose **Aria2**, then select the target storage. When only Aria2 supports the supplied links, it is the only engine available. -Add to `config.toml`: +Configure Aria2 under **Settings → Aria2** in the web console: -```toml -[aria2] -enable = true -url = "http://localhost:6800/jsonrpc" -secret = "your-rpc-secret" # If rpc-secret is configured -remove_after_transfer = true # Remove local files after transfer -``` +- `URL`: Aria2 RPC address, for example `http://localhost:6800/jsonrpc` +- `Secret`: required when `rpc-secret` is configured +- `Keep file`: keep the local file downloaded by Aria2 after the transfer diff --git a/docs/content/en/usage/cli.md b/docs/content/en/usage/cli.md index 216096e9..b5c14621 100644 --- a/docs/content/en/usage/cli.md +++ b/docs/content/en/usage/cli.md @@ -7,7 +7,7 @@ weight: 21 Besides running the Telegram bot with `./saveany-bot` (no subcommand), the binary exposes two helper subcommands for moving local files into a storage backend: `upload` (one-shot) and `watch` (continuous). -These subcommands load the same `config.toml` as the bot, initialize the database and caches, then perform their task. They do **not** start the Telegram bot itself, although storages of type `telegram` will spin up the bot client just for the upload. +These subcommands load the same database settings as the bot, initialize the database and caches, then perform their task. They do **not** start the Telegram bot itself, although storages of type `telegram` will spin up the bot client just for the upload. ## `upload` — Upload a Single File @@ -20,7 +20,7 @@ Flags: | Flag | Required | Description | |---|---|---| | `-f, --file` | Yes | Path to the local file to upload | -| `-s, --storage` | Yes | Target storage name (must exist in `config.toml`) | +| `-s, --storage` | Yes | Target storage name (must exist in the web console storage list) | | `-d, --dir` | No | Destination directory within the storage. Defaults to the storage's `base_path` | | `--no-progress` | No | Disable the terminal progress bar | @@ -82,7 +82,7 @@ Examples: - Relative directory structure is preserved under the destination directory. A file written to `./inbox/sub/file.txt` with `--path ./inbox` is uploaded to `/sub/file.txt`. - `watch` runs until interrupted (e.g. `Ctrl-C` / `SIGINT`); in-flight uploads are drained before exit. -- Retries follow the global `retry` value from `config.toml`, with `--retry-delay` between attempts. +- Retries follow the global `retry` value from Settings, with `--retry-delay` between attempts. - Telegram-type storages will start the bot client automatically to perform uploads. {{< hint warning >}} diff --git a/docs/content/en/usage/config.md b/docs/content/en/usage/config.md index db95eaf9..4c720ff1 100644 --- a/docs/content/en/usage/config.md +++ b/docs/content/en/usage/config.md @@ -5,14 +5,17 @@ weight: 11 # File Naming & Conflict Strategies -SaveAny-Bot lets you customize how saved files are named and how collisions with existing files are resolved, directly in Telegram via the `/config` and `/fnametmpl` commands. +SaveAny-Bot lets you customize how saved files are named and how collisions with existing files are resolved, directly in Telegram with `/config` and `/template`. -## `/config` — User Configuration +## `/config` — User Settings -The `/config` command opens an inline menu where you can change two per-user settings: +The `/config` command contains the following per-user settings: - **Filename strategy** — how the saved file is named - **Duplicate file strategy** — what happens when a file with the same name already exists in the target storage +- **Silent mode** — save incoming files directly to the default location +- **Default storage** — choose the storage (and directory) used by silent mode and unmatched rules +- **Filename template** — view the current template and the `/template` usage Settings are stored per user and apply to all of that user's subsequent save/transfer tasks. @@ -22,7 +25,7 @@ Settings are stored per user and apply to all of that user's subsequent save/tra |---|---| | `Default` | Use the original media filename, or a generated name when no original filename is available | | `Gen From Msg First` | Generate the filename from the message content (e.g. caption, text) and prefer that over the original filename | -| `Template` | Render the filename from a custom template you define with `/fnametmpl` | +| `Template` | Render the filename from a custom template you define with `/template` | ### Duplicate file strategy @@ -37,15 +40,15 @@ Settings are stored per user and apply to all of that user's subsequent save/tra The conflict strategy only kicks in for storage backends that can detect the existence of a file. Backends that do not support existence checks will fall back to overwriting. {{< /hint >}} -## `/fnametmpl` — Custom Filename Template +## `/template` — Custom Filename Template -When the filename strategy is set to `Template`, SaveAny-Bot renders each saved file's name using the template configured via `/fnametmpl`. +When the filename strategy is set to `Template`, SaveAny-Bot renders each saved file's name using the template configured via `/template`. ``` -/fnametmpl [template] +/template [template] ``` -- Running `/fnametmpl` without arguments shows your current template and the help text. +- Running `/template` without arguments shows your current template and the help text. - Running it with a template string sets that template as your filename template. The template uses Go [`text/template`](https://pkg.go.dev/text/template) syntax. The available variables are: @@ -64,12 +67,16 @@ Examples: ``` # Fixed prefix + message id + date -/fnametmpl Image_{{.msgid}}_{{.msgdate}}.jpg +/template Image_{{.msgid}}_{{.msgdate}}.jpg # Use original name if available, otherwise a generated name -/fnametmpl {{.origname}} +/template {{.origname}} ``` +{{< hint info >}} +When an album is saved as a batch (media groups sent to the bot, and watched albums matched by a `NEW-FOR-ALBUM` rule), files without their own caption use the album's caption for template variables and message rules, so `{{.msgtags}}` and `{{.msgraw}}` are populated for every item. If the template produces the same name for several items, later items keep their generated name. +{{< /hint >}} + {{< hint warning >}} The template only takes effect when the filename strategy is set to `Template`. If template parsing fails, SaveAny-Bot falls back to the default filename naming logic. {{< /hint >}} \ No newline at end of file diff --git a/docs/content/en/usage/directlinks.md b/docs/content/en/usage/directlinks.md index 56f04ba5..5f610f8e 100644 --- a/docs/content/en/usage/directlinks.md +++ b/docs/content/en/usage/directlinks.md @@ -1,21 +1,25 @@ --- -title: "Direct Download Links" +title: "URL Downloads" weight: 5 --- -# Direct Download Links +# URL Downloads -Use the `/dl` command to directly download one or more HTTP/HTTPS files to storage. +Use the `/download` command with one or more links and select an engine: + +- **Direct** — plain HTTP/HTTPS download, best for ordinary file links +- **Aria2** — when Aria2 is enabled, including magnet and torrent links +- **Video** — when yt-dlp is available, for supported video sites ```bash -/dl [url2] [url3] ... +/download [url2] [url3] ... ``` Examples: ```bash -/dl https://example.com/file.zip -/dl https://example.com/file1.zip https://example.com/file2.zip +/download https://example.com/file.zip +/download https://example.com/file1.zip https://example.com/file2.zip ``` -The bot will validate the link format and then ask you to select the target storage location. +Choose the engine, then select the target storage. For video sites, yt-dlp options such as `-f best` or `--extract-audio` can be appended to the same command. diff --git a/docs/content/en/usage/parsers.md b/docs/content/en/usage/parsers.md index e256ad02..0196e8e1 100644 --- a/docs/content/en/usage/parsers.md +++ b/docs/content/en/usage/parsers.md @@ -9,7 +9,34 @@ Besides files on Telegram, the bot can also save files from other websites via J > See the [Contributing Parsers](../contribute) document for details. -Just send links that match the requirements of a parser to the bot. Currently built-in parsers include: +Send a link that matches a parser. Built-in parsers: - Twitter - Kemono + +## JavaScript plugin API + +JavaScript plugins register one parser with: + +```js +registerParser({ + metadata: { + name: 'example', + version: '1.0.0', + apiVersion: '1', + description: 'Example plugin', + author: 'you', + }, + canHandle(url) { return url.includes('example.com') }, + parse(url) { + return { + site: 'example', + url, + title: 'Example', + resources: [{ url: 'https://cdn.example.com/file.zip', filename: 'file.zip' }], + } + }, +}) +``` + +`apiVersion` is the plugin ABI version and is independent of the plugin's own `version`. The current ABI is `"1"`; plugins with another value are rejected. The available globals are `console`, `ghttp` (`get`, `getJSON`, `head`), and `playwright`. `parse` returns an `Item` with `resources` containing `url`, `filename`, optional `mime_type`, `extension`, `size`, `hash`, `headers`, and `extra`. diff --git a/docs/content/en/usage/rules.md b/docs/content/en/usage/rules.md index 442b8230..9533868e 100644 --- a/docs/content/en/usage/rules.md +++ b/docs/content/en/usage/rules.md @@ -9,6 +9,8 @@ Storage rules allow you to define redirection rules when the bot uploads files t See: #28 +Send `/rules` to view the current mode and rules, enable or disable rule matching, and delete rules. Add rules and import presets with the commands below. + Currently supported rule types: 1. FILENAME-REGEX @@ -17,39 +19,40 @@ Currently supported rule types: Basic syntax for adding rules: -"RuleType RuleContent StorageName Path" +"RuleType RuleContent StorageID|StorageName Path" -Pay attention to spaces; the bot can only parse correctly formatted syntax. Below is an example of a valid rule command: +A numeric storage argument is treated as a storage ID (shown on the Storages page of the web console); anything else is treated as a storage name. Use an ID when the name contains spaces, or quote the name. Below is an example of a valid rule command: ``` -/rule add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ MyAlist /videos +/rules add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ 3 /videos +/rules add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ "My Alist" /videos ``` -In addition, if `CHOSEN` is used as the storage name in the rule, it means files will be stored under the path of the storage you selected by clicking the inline button. +In addition, if `CHOSEN` is used as the storage in the rule, it means files will be stored under the path of the storage you selected by clicking the inline button. -You can also toggle whether rules are applied with `/rule switch`. When rule mode is off, all files go to the default storage. +You can also toggle whether rules are applied with `/rules switch`. When rule mode is off, all files go to the default storage. ## Preset Rules Manually writing regex rules for common file types is tedious, so the bot ships a built-in set of preset categories (video, image, audio, document, archive) that you can import in one command: ``` -/rule preset [base_path] +/rules preset [base_path] ``` Parameters: -- `storage`: Target storage name (must exist and be accessible to you) +- `storage`: Target storage ID or name (must exist and be accessible to you) - `base_path`: Optional. Each preset category's subdirectory is created under this path. If omitted, the default category directory names are used directly. Examples: ``` # Import preset rules into "MyAlist" with the default directory layout -/rule preset MyAlist +/rules preset MyAlist # Import preset rules with a custom base path "downloads/sorted" -/rule preset MyAlist downloads/sorted +/rules preset MyAlist downloads/sorted ``` This will create `FILENAME-REGEX` rules for each category, routing matched files to the corresponding subdirectory under `base_path`: @@ -63,7 +66,7 @@ This will create `FILENAME-REGEX` rules for each category, routing matched files | archive | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` | {{< hint info >}} -Preset rules are regular `FILENAME-REGEX` rules once imported. You can view, edit, or delete them individually with `/rule` and `/rule del ` like any other rule. +Preset rules are regular `FILENAME-REGEX` rules once imported. You can view or delete them individually from `/rules` like any other rule. {{< /hint >}} Rule types: @@ -76,7 +79,7 @@ Matches based on filename regex. The rule content must be a valid regular expres FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ MyAlist /videos ``` -This means files with extensions mp4, mkv, ts, avi, flv will be saved to the `/videos` directory in the storage named `MyAlist` (also affected by the `base_path` in the configuration file). +This means files with extensions mp4, mkv, ts, avi, flv will be saved to the `/videos` directory in the storage named `MyAlist` (also affected by the `base_path` in storage settings). ## MESSAGE-REGEX diff --git a/docs/content/en/usage/silent.md b/docs/content/en/usage/silent.md index fcd84c9d..53ccf34e 100644 --- a/docs/content/en/usage/silent.md +++ b/docs/content/en/usage/silent.md @@ -3,12 +3,10 @@ title: "Silent Mode" weight: 2 --- -# Silent Mode (silent) +# Silent Mode -Use the `/silent` command to toggle silent mode. +Silent mode is configured in `/config`. Toggle **Silent mode** there. -By default, silent mode is off, and the bot will ask you for the save location of each file. +By default, silent mode is off, and the bot asks for the save location of each file. -When silent mode is enabled, the bot will save files directly to the default location without confirmation. - -Before enabling silent mode, you need to set the default save location using the `/storage` command. +When silent mode is enabled, the bot saves files directly to the default location without confirmation. Set the default storage and directory in `/config` before enabling it. diff --git a/docs/content/en/usage/watch.md b/docs/content/en/usage/watch.md index 2af3d0fc..bc7b2bc3 100644 --- a/docs/content/en/usage/watch.md +++ b/docs/content/en/usage/watch.md @@ -11,18 +11,21 @@ This feature requires enabling UserBot integration. You can watch messages in a specific chat and automatically save them to the default storage, following storage rules. You can also add filters so that only matching messages are saved. -Watch a chat: +Send `/watch` to manage watched chats: -``` -/watch [filter] -``` +- **Add watch** — shows the syntax for adding a chat +- **Watched chats** — lists watched chats and lets you stop each one +- **Notifications** — toggles task notifications for all watched chats +- **Sync peers** — imports the Userbot dialog list into the peer cache -Stop watching: +To start watching a chat, send: ``` -/unwatch +/watch [filter] ``` +To stop watching, open **Watched chats** from `/watch` and select the chat. + Filter types: ## msgre @@ -30,19 +33,13 @@ Filter types: Regex-match the message text. For example: ``` -/watch 12345678 msgre:.*hello.* +/watch -1002229835658 msgre:.*hello.* ``` -This will watch the chat with ID `12345678`, and only save messages whose text contains `hello`. +This will watch the channel with ID `-1002229835658`, and only save messages whose text contains `hello`. Use the Bot API form: positive IDs for users, negative IDs for basic groups, and `-100...` for supergroups and channels. ## Task notifications -Task notifications are disabled by default for every user. Enable them to get a message for every message the bot picks up from a watched chat, updated with the download progress and the final result: - -``` -/watch notify on -``` - -Turn them off again with `/watch notify off`, and query the current state with `/watch notify`. +Task notifications are disabled by default for every user. Enable them with `/watch` → **Notifications** to get a message for every message the bot picks up from a watched chat, updated with the download progress and the final result. Disable them the same way. -Notifications are sent by the bot to your private chat with it. The toggle is per user, not per watched chat: `/watch notify off` stops notifications for every chat that user watches. +Notifications are sent by the bot to your private chat with it. The toggle is per user, not per watched chat. diff --git a/docs/content/en/usage/ytdlp.md b/docs/content/en/usage/ytdlp.md index c1504ed1..faee2a08 100644 --- a/docs/content/en/usage/ytdlp.md +++ b/docs/content/en/usage/ytdlp.md @@ -1,32 +1,32 @@ --- -title: "yt-dlp Video Download" +title: "Video Downloads (yt-dlp)" weight: 7 --- -# yt-dlp Video Download +# Video Downloads (yt-dlp) {{< hint warning >}} This feature requires the yt-dlp command-line tool installed on your system. {{< /hint >}} -Use the `/ytdlp` command to download videos and audio from supported video websites, including YouTube, Bilibili, Twitter, and 1000+ other sites. +Video extraction is available through `/download`. Send a supported video URL, then choose **Video (yt-dlp)**. It supports YouTube, Bilibili, Twitter, and 1000+ other sites. ```bash -/ytdlp [url2] [flags...] +/download [url2] [flags...] ``` Examples: ```bash # Basic download -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ # Download multiple videos -/ytdlp https://www.youtube.com/watch?v=video1 https://www.youtube.com/watch?v=video2 +/download https://www.youtube.com/watch?v=video1 https://www.youtube.com/watch?v=video2 # Use custom parameters -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ -f best -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ --extract-audio --audio-format mp3 +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ -f best +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ --extract-audio --audio-format mp3 ``` Common parameters: @@ -38,12 +38,14 @@ Common parameters: - `--write-sub`: Download subtitles - `--write-thumbnail`: Download thumbnail +For safety, flags that execute programs, load external configuration, or read and write arbitrary files are rejected: `--exec`, `--exec-before-download`, `--config-location`, `--load-info-json`, `--batch-file`, `--downloader`, `--external-downloader`, `--paths`, `--cache-dir`, `--postprocessor-args`, `--netrc-cmd`, `--plugin-dirs`, `--update` and similar. + ## File names Downloaded files are named after the video title (`%(title)s.%(ext)s`) and saved into the storage directory you selected. Pass a yt-dlp output template to customize the name: ```bash -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ -o "%(uploader)s - %(title)s.%(ext)s" +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ -o "%(uploader)s - %(title)s.%(ext)s" ``` Only the template part of `-o/--output` is used: the download directory is always managed by the bot, and any relative path in the template is created below it and kept in the storage path. To change the default for every download, set `filename_template` in the [`[ytdlp]` config](../deployment/configuration); `restrict_filenames` there keeps file names ASCII-only, which drops non-latin characters, spaces and `&`. diff --git a/docs/content/zh/_index.md b/docs/content/zh/_index.md index a8df9052..41e2c3eb 100644 --- a/docs/content/zh/_index.md +++ b/docs/content/zh/_index.md @@ -9,21 +9,21 @@ title: 介绍 ![](https://img.shields.io/github/v/release/krau/SaveAny-Bot?color=cyan&style=flat-square) ![](https://img.shields.io/github/downloads/krau/SaveAny-Bot/total?style=flat-square) -把 Telegram 上的文件转存到多种存储端. +把 Telegram 上的文件转存到多种存储端。 -## 🎯 特性 +## 特性 -- 支持文档/视频/图片/贴纸…甚至还有 [Telegraph](https://telegra.ph/) -- 破解禁止保存的文件 +- 支持文档、视频、图片、贴纸及 [Telegraph](https://telegra.ph/) 图集 +- 支持转存受限频道中禁止转发与保存的内容 - 批量下载 - 流式传输 -- 多用户使用 -- 基于存储规则的自动整理 -- 监听并自动转存指定聊天的消息, 支持过滤 -- 在不同存储端之间转存文件 -- 集成 yt-dlp, 从所支持的网站下载并转存媒体文件 -- 集成 Aria2, 支持直链/磁力下载和转存 -- 使用 js 编写解析器插件以转存任意网站的文件 +- 多用户支持 +- 基于规则自动分类整理 +- 监听指定聊天并按过滤器自动转存 +- 存储端之间互相传输文件 +- 集成 yt-dlp,下载视频并转存 +- 集成 Aria2,支持直链与磁力链接转存 +- 支持编写 JavaScript 解析器插件扩展站点 - 存储端支持: - Alist - S3 diff --git a/docs/content/zh/contribute/_index.md b/docs/content/zh/contribute/_index.md index 95970a3e..82bd5dbc 100644 --- a/docs/content/zh/contribute/_index.md +++ b/docs/content/zh/contribute/_index.md @@ -5,30 +5,31 @@ weight: 20 # 参与开发 -在开始之前, 请 Fork 本项目, 并克隆到本地, 安装好 Go 开发环境. +在开始之前,请 Fork 本项目并克隆到本地,准备好 Go 开发环境。 -以下是一些贡献代码的指南或建议, 你不必完全遵守, 但将有助于快速 review 并合并你的提交: +贡献代码建议遵循以下约定,以便快速合并: -- **新功能请先提交 Issue**, 以便讨论设计和实现细节, 并避免因与项目设计不符而被拒绝. -- **使用现代开发工具**, 确保提交前格式化代码, 并保持风格一致. -- **使用[语义化提交](https://www.conventionalcommits.org/zh-hans/v1.0.0/)**, 避免提交消息模糊或过于简单. +- **新功能请先提交 Issue**:便于提前讨论设计细节,避免因与整体规划不符而被拒绝。 +- **代码风格**:提交前请格式化代码并保持项目整体风格一致。 +- **提交规范**:遵循[语义化提交](https://www.conventionalcommits.org/zh-hans/v1.0.0/)规范。 +- **测试验证**:参阅[测试文档](testing),运行测试验证改动。 ## 贡献新存储端 -1. 在 `pkg/enums/storage/storages.go` 中添加新的存储端类型, 并运行代码生成 -2. 在 `config/storage` 目录下定义存储端配置, 并添加到 `config/storage/factory.go` 中 -3. 在 `storage` 目录下新建一个包, 编写存储端实现, 然后在 `storage/storage.go` 中导入并添加它 -4. 更新文档, 添加配置说明 +1. 在 `pkg/enums/storage/storages.go` 中添加新的存储端类型,并运行代码生成 +2. 在 `config/storage` 目录下定义存储端配置,并添加到 `config/storage/factory.go` 中 +3. 在 `storage` 目录下新建一个包编写存储端实现,然后在 `storage/storage.go` 中导入注册 +4. 更新文档,添加配置说明 ## 贡献新解析器 -你可以选择使用 Go 编写原生的解析器实现(推荐), 或是使用 JavaScript 以插件的方式实现. +解析器可通过 Go 原生编写(推荐),也可编写为 JavaScript 插件。 -如果使用 Go 编写, 请: +若使用 Go 编写: -1. 在 `parsers` 目录下新建一个包, 编写解析器实现 +1. 在 `parsers` 目录下新建包编写解析器实现 2. 在 `parsers/parser.go` 的 `init` 中注册解析器 -如果使用 JavaScript 编写, 请参考 `plugins/example_parser_basic.js` 的实现, 并在该文件夹下新建一个 js 文件, 实现你的解析逻辑. +若使用 JavaScript 编写,请参考 `plugins/example_parser_basic.js` 并在该文件夹下新建 `.js` 文件实现解析逻辑。 -需要注意, `plugins` 目录下解析器默认不会被编译到二进制文件中, 用户需要手动下载它们并放到本地指定目录下以启用它们. \ No newline at end of file +注意:`plugins` 目录下的解析器不打包进二进制文件,需放置于配置指定的插件目录中加载。 \ No newline at end of file diff --git a/docs/content/zh/contribute/testing.md b/docs/content/zh/contribute/testing.md new file mode 100644 index 00000000..ba7c88a5 --- /dev/null +++ b/docs/content/zh/contribute/testing.md @@ -0,0 +1,60 @@ +--- +title: "测试" +weight: 10 +--- + +# 测试 + +项目提供预置测试数据的本地环境,以及无需 Telegram 凭据的端到端测试。 + +## 预置环境 + +```bash +make testenv +``` + +命令会创建临时数据目录并输出: + +- Web UI 地址 +- 管理员密码(默认为 `saveany-test`) +- 预置的本地存储和白名单用户 +- 用于创建直链下载任务的示例 URL + +使用 `--keep` 保留生成的数据目录,或传入 `--data-dir` 复用已有目录。Telegram 保持停止状态,可直接测试 Web 控制台及非 Telegram 任务流程。 + +## 后端端到端测试 + +```bash +make e2e-api +``` + +测试会构建二进制,在随机端口启动预置环境并调用公开 API。覆盖认证与 CSRF、新手引导、存储与用户生命周期、设置更新、SPA 静态资源回退,以及下载直链文件到预置存储的任务。 + +测试位于 `e2e/`,使用 `e2e` build tag,普通 `go test ./...` 不会触发。 + +## Telegram 端到端测试 + +```bash +make e2e-telegram +``` + +该测试让 Bot handler、dispatcher、任务队列和存储实现运行在内存 `tg.Invoker` 之上。覆盖 `/start`、群聊准入、将 Telegram 文件保存到本地存储(含进度编辑)及上传到 Telegram 存储。全程无需真实 Telegram 账号或外部网络。 + +## 前端端到端测试 + +```bash +make e2e-web-setup # 首次运行, 下载 Chromium +make e2e-web +``` + +启动预置环境并使用 Playwright 驱动 Web UI。覆盖登录、存储配置测试与创建,以及直链任务创建与完成状态校验。测试产物写入 `tmp/e2e-web-results/`。 + +Playwright 为 `web/ui` 的开发依赖,版本由 `pnpm-lock.yaml` 锁定。`make e2e-web-setup` 会自动安装匹配的 Chromium。 + +## 完整测试 + +```bash +make e2e +``` + +依次运行 API、Telegram 和前端浏览器测试(前端测试前会自动构建 Web UI)。假 Telegram 层覆盖了命令、下载、上传与群聊准入;若需测试线上 Telegram 真实行为、Userbot 登录及限流,仍需使用测试账号。 diff --git a/docs/content/zh/deployment/configuration/_index.md b/docs/content/zh/deployment/configuration/_index.md index dccc99c1..47fd7e41 100644 --- a/docs/content/zh/deployment/configuration/_index.md +++ b/docs/content/zh/deployment/configuration/_index.md @@ -1,270 +1,106 @@ --- title: "配置说明" +weight: 11 --- # 配置说明 -SaveAnyBot 的配置文件使用 toml 格式, 你可以在 [TOML 官方网站](https://toml.io/) 上了解更多关于 toml 的语法. - -SaveAnyBot 需要读取工作目录下的 `config.toml` 文件作为配置文件, 若缺少该文件则会创建默认文件, 并尝试从环境变量中加载配置. - -以下是一个最简的配置文件示例: - -```toml -[telegram] -token = "1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZ" - -[[users]] -# telegram user id -id = 777000 -blacklist = true - -[[storages]] -name = "本机存储" -type = "local" -enable = true -base_path = "./downloads" -``` - -## 详细配置 - -### 全局配置 - -- `lang`: Bot 使用的语言, 默认为 `zh-CN` (简体中文), 设为 `en` 则使用英语. -- `stream`: 是否启用 Stream 模式, 默认为 `false`. 启用后 Bot 将直接将文件流式传输到存储端(若存储端支持), 不需要下载到本地 -{{< hint warning >}} -Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一些弊端: -
-
    -
  • 无法使用多线程从 Telegram 下载文件, 速度较慢.
  • -
  • 网络不稳定时, 任务失败率高.
  • -
  • 无法在中间层对文件进行处理, 例如自动文件类型识别.
  • -
  • 并非支持所有存储端, 不支持的存储端可能会降级为普通模式或无法上传.
  • -
-{{< /hint >}} -- `workers`: 同时处理任务数量, 默认为 3 -- `threads`: 下载文件时使用的线程数, 默认为 4. 仅在未启用 Stream 模式时生效. -- `retry`: 任务失败时的重试次数, 默认为 3. -- `proxy`: 全局代理配置, 配置后程序内一切网络连接将会尝试使用该代理, 可选. - -```toml -lang = "zh-CN" -stream = false -workers = 3 -threads = 4 -retry = 3 -proxy = "socks5://127.0.0.1:7890" -``` - -### Telegram 配置 - -- `token`: 你的 Telegram Bot Token, 可以通过 [BotFather](https://t.me/botfather) 创建 Bot 并获取 Token. -- `app_id`, `app_hash`: Telegram API ID & Hash, 在 [Telegram API](https://my.telegram.org/apps) 创建应用获取, 若不提供则使用默认值. -- `flood_retry`: Flood 控制重试次数, 默认为 5. -- `rpc_retry`: RPC 请求重试次数, 默认为 5. -- `proxy`: 代理配置, 可选. - - `enable`: 是否启用代理. - - `url`: 代理地址 -- `userbot`: userbot 配置, 可选. - - `enable`: 启用 userbot 集成, 需要登录用户账号, 此时请务必使用自己的 api id & hash. - - `session`: userbot 会话文件路径, 默认为 `data/usersession.db`. - -{{< hint warning >}} -启用 userbot 集成后, bot 可以下载私密频道和群组的文件, 但具有无法避免的账号被封禁的风险. -
-开启 userbot 集成后第一次启动 bot 时需要通过终端交互输入手机号, 2FA 和验证码. -
-如果你使用 docker 部署, 请使用 -it 参数为容器提供交互式环境, 然后执行登录操作. -{{< /hint >}} - -```toml -[telegram] -token = "1234567890:ABCDEFGHIJKLMNOPQRSTUVWXYZ" -app_id = 1025907 -app_hash = "452b0359b988148995f22ff0f4229750" -flood_retry = 5 -rpc_retry = 5 -[telegram.proxy] -enable = false -url = "socks5://127.0.0.1:7890" -[telegram.userbot] -enable = false -session = "data/usersession.db" -``` - -### Aria2 配置 - -Aria2 是一个强大的下载管理器,支持 HTTP/HTTPS、FTP、BitTorrent 等多种协议。启用后,Bot 可以使用 `/aria2dl` 命令通过 Aria2 下载文件。 - -- `enable`: 是否启用 Aria2 支持,默认为 `false` -- `url`: Aria2 RPC 地址,通常为 `http://localhost:6800/jsonrpc` -- `secret`: Aria2 RPC 密钥,如果你在 Aria2 中配置了 `rpc-secret`,需要在此填写 -- `remove_after_transfer`: 转存完成后是否删除 Aria2 下载的本地文件,默认为 `true` - -{{< hint info >}} -Aria2 需要单独安装和运行。你可以参考 [Aria2 官方文档](https://aria2.github.io/) 了解如何安装和配置 Aria2。 -{{< /hint >}} - -```toml -[aria2] -enable = true -url = "http://localhost:6800/jsonrpc" -secret = "your-rpc-secret" -remove_after_transfer = true -``` - -### yt-dlp 配置 - -用于配置 `/ytdlp` 命令以及 HTTP API 中 `ytdlp` 任务类型在未传自定义参数时的默认行为. - -- `max_height`: 默认下载的最高视频清晰度 (按高度限制), 如 `1080`, `720`, `480`; `0` 表示不限制 (下载最佳画质). 当设置了 `format` 时此项被忽略. -- `format`: 直接指定 yt-dlp format 选择表达式, 设置后优先级高于 `max_height`, 例如 `bv*[height<=720]+ba/b`. -- `recode`: 下载后转封装的视频容器格式 (如 `mp4`), 留空则不转封装. -- `filename_template`: 下载文件名模板 (yt-dlp output template), 默认 `%(title)s.%(ext)s`. 模板中的目录会创建在 bot 的下载目录下, 并保留到存储路径中. -- `restrict_filenames`: 将文件名限制为 ASCII 字符 (yt-dlp `--restrict-filenames`), 同时会去掉空格和 `&`. 默认关闭. - -{{< hint info >}} -`max_height`、`format` 和 `recode` 仅在使用 `/ytdlp` 命令 (或 API 的 `ytdlp` 任务类型) 且未传任何自定义参数时生效. `filename_template` 和 `restrict_filenames` 在传了其他参数时依然生效, 除非自己传了对应的参数 (`-o/--output`、`--restrict-filenames` 或 `--no-restrict-filenames`). -{{< /hint >}} - -```toml -[ytdlp] -max_height = 1080 -format = "" # 例如 "bv*[height<=720]+ba/b" -recode = "mp4" # 留空则不转封装 -filename_template = "%(title)s.%(ext)s" # 例如 "%(uploader)s - %(title)s.%(ext)s" -restrict_filenames = false -``` - -### HTTP API 配置 - -启用后, SaveAny-Bot 会暴露一套 HTTP API, 用于以编程方式创建/查询/取消任务. 完整的接口说明见 [HTTP API](../../usage/api). - -- `enable`: 是否启用 HTTP API 服务, 默认为 `false`. -- `host`: 监听地址, 默认 `0.0.0.0`. -- `port`: 监听端口, 默认 `8080`. -- `token`: 鉴权 Token, **强烈建议设置** — 若为空, API 将在无任何鉴权的情况下暴露. - -```toml -[api] -enable = false -host = "0.0.0.0" -port = 8080 -token = "your-token" -``` - -### 日志配置 - -- `level`: 日志级别, 可选 `debug`, `info`, `warn`, `error`, `fatal`. 默认为 `info`. - -```toml -[log] -level = "info" -``` - -### 存储端列表 - -存储端列表用于定义 Bot 支持的存储位置, 每个存储端需要指定名称、类型和相关配置, 使用双中括号语法 `[[storages]]` 定义. - -每一个存储端至少需要以下字段: - -- `name`: 存储端名称, 用于在 Bot 中识别, 需要唯一 -- `enable`: 是否启用该存储端, 默认为 `true` -- `type`: 存储端类型, 目前支持以下类型: - - `local`: 本地磁盘 - - `alist`: Alist - - `webdav`: WebDAV - - `s3`: aws S3 及其他兼容 S3 的服务 - - `rclone`: 调用 rclone 实现上传 - - `telegram`: 上传到 Telegram - -示例, 这是一个包含本地存储和 webdav 存储的配置: - -```toml -[[storages]] -name = "本地存储" -type = "local" -enable = true -# 以下是 local 类型存储的自定义配置 -base_path = "./downloads" - -[[storages]] -name = "WebDAV" -type = "webdav" -enable = true -# 以下是 webdav 类型存储的自定义配置 -url = "https://example.com/webdav" -base_path = "/path/to/webdav" -username = "your_username" -password = "your_password" -``` - -所有存储端的自定义配置项可查看 [存储端配置](./storages) - -### 用户列表 - -用户列表用于定义对存储端的访问控制, 每个用户需要指定 Telegram 上的用户 ID, 使用双中括号语法 `[[users]]` 定义. - -- `id`: 用户的 Telegram User ID -- `storages`: 过滤的存储端列表, 使用存储端名称定义, 默认为白名单模式 (即只允许访问列表中的存储端) -- `blacklist`: 是否启用黑名单模式, 默认为 `false`. 若启用黑名单模式, 则仅允许访问**没有**在列表中的存储端. - -示例, 这是一个包含三个用户的配置, 用户 `123123` 只能访问本地存储, 用户 `456456` 只能访问除 WebDAV 以外的存储, 用户 `789789` 启用黑名单模式但没有指定存储端, 因此可以访问所有存储: +配置保存在数据目录下的 SQLite 数据库(默认 `data/saveany.db`),通过 Web 控制台统一管理。首次启动采用默认设置,并在日志中输出带 Token 的直达链接;可通过 `SAVEANY_ADMIN_PASSWORD` 预设管理员密码。 -```toml -[[users]] -id = 123123 -storages = ["本地存储"] - -[[users]] -id = 456456 -storages = ["WebDAV"] -blacklist = true - -[[users]] -id = 789789 -storages = [] -blacklist = true -``` +## 配置页面 -### 事件触发 - -事件触发提供了在 Bot 处理任务时根据任务状态执行自定义操作的能力, 目前仅支持任意命令执行. 使用 `[hook.exec]` 配置. +| 页面 | 内容 | +|---|---| +| 指南 | 首次配置检查清单和日常使用说明 | +| Telegram | Bot Token、API ID/Hash、代理、Userbot 登录 | +| 存储 | 本地磁盘、WebDAV、Alist、S3、Rclone、Telegram 等存储端 | +| 用户 | Telegram 用户 ID、默认存储、目录、规则和存储访问权限 | +| 自动化 | 按用户管理目录、规则、监听聊天和对话同步 | +| 设置 | 任务并发、语言、代理、媒体类型过滤、Aria2、yt-dlp、解析器、缓存、日志、Web 和 API | +| 解析器 | 安装 JavaScript 解析器插件 | +| 系统 | 运行信息、重启、版本检查和更新 | -目前具有以下几种事件类型: +部分配置需重启进程生效(如任务并发数、全局代理、缓存、插件、Hook 及 Web 监听地址),保存时页面会明确提示。 -- `task_before_start`: 任务即将开始前 -- `task_success`: 任务成功完成后 -- `task_fail`: 任务失败后 -- `task_cancel`: 任务被取消后 +### 自动化 -提供的配置值需要为完整的命令行命令, Bot 会在事件发生时执行该命令. 示例: - -```toml -[hook.exec] -task_before_start = "echo '任务即将开始'" -task_success = "bash /path/to/success_script.sh" -task_fail = "curl -X POST https://example.com/api/notify -d '任务失败'" -task_cancel = "bash /path/to/cancel_script.sh" -``` +自动化页面按用户分别管理目录、规则、监听聊天及 Userbot 对话同步。可设置默认目录,并支持规则的启停、增删与预设导入。 ### 解析器 -解析器为 Bot 提供了处理非 Telegram 文件的能力, 例如从其他网站下载文件. 使用 `[parsers]` 配置. +解析器页面用于安装 JavaScript 解析器插件(需在设置中启用解析器插件)。支持上传 `.js` 文件或直接粘贴代码,安装后立即可用,无需重启。 + +### Telegram + +Bot Token 在 [@BotFather](https://t.me/BotFather) 创建。`API ID` 和 `API Hash` 已提供默认值,通常无需修改;启用 Userbot 时建议使用自己的 [Telegram API](https://my.telegram.org/apps) 凭据。 + +保存 Bot 配置后会自动建立连接。Userbot 的登录在 Telegram 页面完成,无需进入终端。 + +### 辅助 Bot + +配置辅助 Bot 可提升 Telegram 文件的传输效率。在 **Telegram** 页面的 **辅助 Bot** 区域填入 Token,将辅助 Bot 加入目标 Telegram 聊天并授予发文件权限。Telegram 存储上传将自动分配给可用且在群内的辅助 Bot;若辅助 Bot 无法访问目标聊天,上传自动回退到主 Bot。辅助 Bot 不响应命令。 + +若需通过多个辅助 Bot 分块加速下载大文件,可在 **Telegram** 页面填写私有的 **辅助下载频道 ID**。需确保主 Bot 可向该频道发帖,且频道内已加入至少两个辅助 Bot。主 Bot 会把收到的文件转发到辅助下载频道,各辅助 Bot 刷新文件引用后下载各自的分块。该功能仅在非流式模式下对达到 **Relay 最小文件大小**(默认 8 MB)的文件生效;频道或辅助 Bot 异常时自动回退为单客户端下载。默认保留转发到频道的消息;关闭“保留频道转发消息”将在下载完成后自动清理。 + +传输并发可在 **设置 → 核心** 中调整: + +| 设置 | 默认值 | 说明 | +|---|---|---| +| `单文件下载线程数` | 4 | 自动调度的基础线程数 | +| `下载线程覆盖` | 0 | 0 表示自动;大于 0 时直接使用该总线程数 | +| `自动线程上限` | 16 | 自动调度时的总线程上限 | +| `全局传输并发上限` | 0 | 0 表示 `workers × threads`;下载与上传共享 | +| `Relay 最小文件大小 (MB)` | 8 | 更小的文件不使用辅助 Bot | +| `分块重试次数` | 20 | 单个分块超时或连接中断的最大重试次数 | + +### 媒体过滤 + +在 **设置 → 媒体过滤** 中可以限制 Bot 只处理指定类型的 Telegram 媒体。`允许的媒体类型` 每行填写一个值,留空表示全部允许: + +| 值 | 匹配内容 | +|---|---| +| `image` | Telegram 图片、图片文件、贴纸和动态图片 | +| `video` | Telegram 视频文件(含视频消息) | +| `audio` | Telegram 音频和语音消息 | +| `document` | 其余文档文件 | + +过滤在创建下载任务之前执行,不匹配的媒体不会下载,也不会出现在任务列表。媒体组只保留匹配项;如果整组都被过滤,则静默跳过。该设置是实例级配置,修改后立即生效。 + +目前只覆盖 Telegram 媒体入口:直接发送/转发媒体、媒体组、`/save` 批量保存、消息链接、监听聊天和 Telegraph 图片。解析器、HTTP 直链和 yt-dlp 等非 Telegram 媒体不参与该过滤。存储规则只负责改变归档位置,不会跳过文件。 + +### 消息元数据 + +在 **设置 → 消息元数据** 中启用后,新的 Telegram 文件任务会保存规范化后的消息数据。本地数据库记录来源聊天、消息 ID、时间、发送者、正文或 caption、回复/转发引用、相册分组及媒体属性;任务详情 API 和 Web 控制台可导出对应 JSON。 + +“保存消息文本”控制是否保存正文与标签。该选项不保存 access hash、Token 或原始 Telegram 对象,且仅对启用后创建的新任务生效。 + +“写入旁车 JSON”会在实际文件旁写入 `<文件路径>.meta.json`,默认关闭。Telegram 存储不会上传旁车文件,而是可用存储配置中的选项消费元数据来还原 caption 或媒体组。 + +### 存储 + +存储端在 **存储** 页面添加并启用。本地磁盘只需要一个基础路径: + +- Docker 部署通常填写 `/app/downloads`。 +- 二进制部署可以填写 `./downloads` 或绝对路径。 + +存储端字段说明见 [存储端配置](./storages)。 + +### 用户 + +只有 **用户** 页面中启用的 Telegram User ID 能使用 Bot。新建用户时至少填写数字 User ID;默认存储可以留空,之后再设置。 -```toml -[parsers] -plugin_enable = true # 是否启用解析器插件 -plugin_dirs = ["./plugins"] # 插件目录, 可以是多个目录 -``` +## 环境变量 -上述两个配置项只用于控制以 JavaScript 编写的解析器插件, Bot 还有内置的使用 Go 实现的解析器, 目前默认开启. +| 变量 | 说明 | +|---|---| +| `SAVEANY_DATA_DIR` | 数据目录,默认 `./data` | +| `SAVEANY_DB_PATH` | SQLite 文件路径,默认 `/saveany.db` | +| `SAVEANY_LISTEN` | Web 监听地址,默认 `127.0.0.1:28080`;需要对外开放时使用 `0.0.0.0:28080` | +| `SAVEANY_ADMIN_PASSWORD` | 首次启动时设置备用管理员密码;未设置则自动生成并打印 | -### 杂项 +## 安全 -```toml -no_clean_cache = false # 是否在退出时不清空缓存文件夹 -# 临时下载文件夹配置 -[temp] -base_path = "./cache" -``` \ No newline at end of file +- Web 控制台默认仅监听 `127.0.0.1:28080`;若需对外开放,请使用反向代理和 HTTPS。 +- 设置中的 `访问 Token` 可用于 `/?token=...` 直接登录 Web UI,也用于 `/api/v1` 的 Bearer 认证。留空时,每次启动自动生成并打印到日志。 +- 首次使用密码登录后,立即修改自动生成的备用管理员密码。 +- 不要公开 `data/`、Bot Token、Userbot 会话或访问 Token。 diff --git a/docs/content/zh/deployment/configuration/storages.md b/docs/content/zh/deployment/configuration/storages.md index 4dce9462..0159a746 100644 --- a/docs/content/zh/deployment/configuration/storages.md +++ b/docs/content/zh/deployment/configuration/storages.md @@ -4,15 +4,17 @@ title: "存储端配置" # 存储端配置 -请先阅读 [配置说明](../) 了解配置文件的基本格式. +在 Web 控制台的 **存储** 页面添加存储时填写以下字段。 + +存储名称支持空格、中文与常用符号(最多 64 字符,不可含控制字符)。在 `/dirs`、`/transfer` 等命令中引用含空格的名称时需加双引号。名称可在 Web 控制台随时更改,用户默认存储、目录及规则均基于内部存储 ID 关联,不受重命名影响。 ## Alist `type=alist` -不支持 Stream 模式. +不支持流式下载(Stream 模式)。 -```toml +```text url = "https://alist.example.com" # Alist 的 URL username = "your_username" # Alist 的用户名 password = "your_password" # Alist 的密码 @@ -27,14 +29,14 @@ token = "your_token" `type=local` -```toml +```text base_path = "./downloads" # 本地存储的基础路径, 所有文件将存储在此路径下 ``` ## WebDAV `type=webdav` -```toml +```text url = "https://webdav.example.com" # WebDAV 的 URL username = "your_username" # WebDAV password = "your_password" # WebDAV 的密码 @@ -45,8 +47,8 @@ base_path = "/path/to/webdav" # WebDAV 中的基础路径, 所有文件将存储 `type=s3` -```toml -endpoint = "s3.example.com" # S3 的端点, 默认为 aws S3 的端点 +```text +endpoint = "https://s3.example.com" # S3 的端点, 默认为 aws S3 的端点 region = "us-east-1" # S3 的区域 access_key_id = "your_access_key_id" # S3 的访问密钥 ID secret_access_key = "your_secret_access_key" # S3 的秘密访问密钥 @@ -67,36 +69,42 @@ https://your_bucket_name.s3.example.com/path/to/s3/your_file https://s3.example.com/your_bucket_name/path/to/s3/your_file ``` -如果你使用的是第三方的兼容 S3 的服务, 一般使用的是路径风格的 URL. 而 AWS S3 则通常使用虚拟主机风格的 URL. 详情请参考你所使用的 S3 兼容服务的文档. +第三方 S3 兼容服务通常采用路径风格 URL,AWS S3 则默认使用虚拟主机风格。具体请参考对应服务文档。 ## Telegram `type=telegram` -不支持 Stream 模式. +不支持流式下载(Stream 模式)。 -```toml +```text # Telegram 聊天 ID, Bot 将把文件发送到这个聊天 -chat_id = "123456789" +chat_id = "-1001234567890" # Bot API 聊天 ID(用户为正数,普通群为负数,超级群/频道使用 -100...) # 是否强制使用文件方式发送, 默认为 false force_file = false # 是否跳过大文件, 默认为 false. 如果启用, 超过 Telegram 限制的文件将不会上传. skip_large = false -# 超限视频是否使用 FFmpeg 无损分割成一个媒体组(最多 10 个)内可独立播放的小视频;失败或需要超过 10 段时回退到 ZIP 分卷. +# 超限视频是否使用 FFmpeg 分割为单组(最多 10 个)独立播放的小视频;失败或超过 10 段时回退为 ZIP 分卷. split_large_video = false # 分卷大小, 单位 MB. 设为 0 时使用实际上传账号的限制:Bot/普通用户为 2000 MB,Premium 用户为 4000 MB. # 超过该大小的文件将被分割成多个部分上传;非视频使用 ZIP 格式. # 当 skip_large 启用时, 该选项无效. split_size_mb = 0 +# 没有实时 caption 时,用元数据中的消息文本作为发送 caption +use_metadata_caption = false +# 没有实时分组信息时,用元数据 grouped_id 将文件作为媒体组发送 +use_metadata_album = false ``` +启用消息元数据后,Telegram 存储会直接消费元数据,不会把 `.meta.json` 作为文件上传。`use_metadata_caption` 在没有实时 caption 时用原消息文本还原 caption;`use_metadata_album` 在没有实时分组信息时用元数据 `grouped_id` 将文件作为媒体组发送。两项默认关闭。媒体组只会在第一个文件上添加 caption,以符合 Telegram 客户端行为。 + ## Rclone `type=rclone` 通过 [rclone](https://rclone.org/) 命令行工具支持多种云存储服务. 需要先安装 rclone 并配置好远程存储. -```toml +```text # rclone 配置的远程名称, 可以是任何在 rclone.conf 中配置的远程 remote = "mydrive" # 在远程存储中的基础路径, 所有文件将存储在此路径下 @@ -109,22 +117,15 @@ flags = ["--transfers", "4", "--checkers", "8"] ### 配置 rclone 远程 -首先需要配置 rclone 远程, 运行 `rclone config` 命令进行交互式配置, 或直接编辑 `rclone.conf` 文件. +首先需要配置 rclone 远程,运行 `rclone config` 进行交互式配置,或直接编辑 `rclone.conf` 文件。 -rclone 支持多种云存储服务, 包括但不限于: -- Google Drive -- Dropbox -- OneDrive -- Amazon S3 及兼容服务 -- SFTP -- FTP -- 更多服务请参考 [rclone 官方文档](https://rclone.org/overview/) +支持 Google Drive、Dropbox、OneDrive、S3 兼容存储、SFTP/FTP 等服务,详见 [rclone 官方文档](https://rclone.org/overview/)。 ### 使用示例 配置 Google Drive 后, 可以这样配置存储: -```toml +```text [[storages]] name = "GoogleDrive" type = "rclone" @@ -135,7 +136,7 @@ base_path = "/SaveAnyBot" 如果使用自定义的 rclone 配置文件: -```toml +```text [[storages]] name = "MyRemote" type = "rclone" diff --git a/docs/content/zh/deployment/installation.md b/docs/content/zh/deployment/installation.md index f52fe31f..0e85b593 100644 --- a/docs/content/zh/deployment/installation.md +++ b/docs/content/zh/deployment/installation.md @@ -1,30 +1,55 @@ --- title: "安装与更新" +weight: 10 --- # 安装与更新 -## 从预编译文件部署(推荐) +准备空数据目录即可启动,随后在 Web 控制台完成 Bot、存储和用户配置。 -在 [Release](https://github.com/krau/SaveAny-Bot/releases) 页面下载对应平台的二进制文件. +## 从预编译文件部署 -在解压后目录新建 `config.toml` 文件, 参考 [配置说明](../configuration) 编辑配置文件 - -运行: +在 [Release](https://github.com/krau/SaveAny-Bot/releases) 页面下载对应平台的二进制文件并解压: ```bash chmod +x saveany-bot ./saveany-bot ``` +程序默认使用当前目录下的 `data/` 保存数据库和会话。Web UI 默认监听 `127.0.0.1:28080`,启动时会输出带访问 Token 的直达地址: + +```text +Web UI available at http://127.0.0.1:28080/?token=... +``` + +打开该地址即可登录。未在“设置 → Web”中指定访问 Token 时,每次启动均会重新生成。 + +如需在其他网卡或端口访问,通过命令行指定监听地址: + +```bash +./saveany-bot --listen 0.0.0.0:28080 +``` + +对应的环境变量为 `SAVEANY_LISTEN`: + +```bash +SAVEANY_LISTEN='0.0.0.0:28080' ./saveany-bot +``` + +自动生成的管理员密码用于首次登录,也可通过环境变量 `SAVEANY_ADMIN_PASSWORD` 预设。初始密码仅在首次启动时打印,登录后可在侧栏修改密码。打开日志中的直达地址后,参照指南页面完成初始化: + +1. 在 Telegram 页面填写 Bot Token。 +2. 在存储页面添加至少一个存储。 +3. 在用户页面添加你的 Telegram User ID。 + ### 进程守护 {{< tabs "daemon" >}} {{< tab "systemd (常规 Linux)" >}} -创建文件 /etc/systemd/system/saveany-bot.service 并写入以下内容: +创建 `/etc/systemd/system/saveany-bot.service`: -{{< codeblock >}} +```ini [Unit] Description=SaveAnyBot After=systemd-user-sessions.service @@ -37,133 +62,97 @@ Restart=always [Install] WantedBy=multi-user.target -{{< /codeblock >}} - -设为开机启动并启动服务: +``` -{{< codeblock >}} +```bash systemctl enable --now saveany-bot -{{< /codeblock >}} +``` {{< /tab >}} - {{< tab "procd (OpenWrt)" >}} -

添加开机自启动服务

- -创建文件 /etc/init.d/saveanybot ,参考 wrt_init 并自行修改: +创建 `/etc/init.d/saveanybot`,参考 [wrt_init](https://github.com/krau/SaveAny-Bot/blob/main/docs/confs/wrt_init) 并修改工作目录与程序路径: -{{< codeblock >}} +```sh #!/bin/sh /etc/rc.common - -#This is the OpenWRT init.d script for SaveAnyBot - -START=99 +START=99 STOP=10 description="SaveAnyBot" - WORKING_DIR="/mnt/mmc1-1/SaveAnyBot" EXEC_PATH="$WORKING_DIR/saveany-bot" -start() { - echo "Starting SaveAnyBot..." - cd $WORKING_DIR - $EXEC_PATH & -} -stop() { - echo "Stopping SaveAnyBot..." - killall saveany-bot -} -reload() { - stop - start -} - -{{< /codeblock >}} - -赋予权限: - -{{< codeblock >}} -chmod +x /etc/init.d/saveanybot -{{< /codeblock >}} - -然后将文件复制到 /etc/rc.d 并重命名为 S99saveanybot, 同样赋予权限: - -{{< codeblock >}} -chmod +x /etc/rc.d/S99saveanybot -{{< /codeblock >}} - -

添加快捷指令

- -创建文件 /usr/bin/sabot ,参考 wrt_bin 并自行修改,注意此处文件编码仅支持 ANSI 936 . - -随后赋予权限: - -{{< codeblock >}} -chmod +x /usr/bin/sabot -{{< /codeblock >}} +start() { cd $WORKING_DIR && $EXEC_PATH & } +stop() { killall saveany-bot; } +``` -使用: sudo sabot start|stop|restart|status|enable|disable +```bash +chmod +x /etc/init.d/saveanybot +ln -s /etc/init.d/saveanybot /etc/rc.d/S99saveanybot +``` {{< /tab >}} {{< /tabs >}} - ## 使用 Docker 部署 ### Docker Compose -下载 [docker-compose.yml](https://github.com/krau/SaveAny-Bot/blob/main/docker-compose.yml) 文件, 在同目录下新建 `config.toml` 文件, 参考 [config.example.toml](https://github.com/krau/SaveAny-Bot/blob/main/config.example.toml) 编辑配置文件. - -启动: +下载 [docker-compose.yml](https://github.com/krau/SaveAny-Bot/blob/main/docker-compose.yml),按需修改端口和数据目录,然后启动: ```bash docker compose up -d +docker compose logs -f saveany-bot +``` + +默认挂载: + +- `./data:/app/data` — 设置、任务记录和会话 +- `./downloads:/app/downloads` — 本地存储的默认目录 +- `./cache:/app/cache` — 下载缓存 + +可指定管理员密码;未指定时从首启日志读取生成的密码。使用桥接网络时需将监听地址设为 0.0.0.0: + +```yaml +environment: + SAVEANY_ADMIN_PASSWORD: change-me + SAVEANY_LISTEN: 0.0.0.0:28080 ``` ### Docker -```shell +```bash docker run -d --name saveany-bot \ - -v /path/to/config.toml:/app/config.toml \ - -v /path/to/downloads:/app/downloads \ + -p 28080:28080 \ + -e SAVEANY_LISTEN=0.0.0.0:28080 \ + -v ./data:/app/data \ + -v ./downloads:/app/downloads \ + -v ./cache:/app/cache \ ghcr.io/krau/saveany-bot:latest ``` -{{< hint info >}} -关于 docker 镜像的变体版本 -
-
    -
  • 默认版本: 包含所有功能和依赖, 体积较大. 如果没有特殊需要, 请使用此版本
  • -
  • micro: 精简版本, 去除部分可选依赖, 体积较小
  • -
  • pico: 极简版本, 仅包含核心功能, 体积最小
  • -
-你可以根据需要, 通过指定不同的标签来拉取合适的版本, 例如: ghcr.io/krau/saveany-bot:micro -
-关于变体版本的更详细的区别, 请参考项目根目录下的 Dockerfile 文件. -{{< /hint >}} +打开日志中输出的直达链接。若使用本地存储,容器内路径可设为 `/app/downloads`。 ## 更新 -若使用预编译二进制文件部署, 使用以下 CLI 命令更新: +### 预编译二进制 ```bash ./saveany-bot up ``` -该命令会就地替换二进制文件; 已运行的 Bot 会在重启后才使用新版本. 也可以在 Telegram 中发送 `/update`, Bot 会自行下载新版本、替换自身二进制并就地重启进程 (Windows 上无法就地重启, 进程会退出, 由服务管理器启动新版本). +或在 Telegram 中发送 `/update`。更新后进程会自动重启(Windows 环境需手动或由服务管理器重启)。 -如果是 Docker 部署, 使用以下命令更新: - -docker: +### Docker ```bash docker pull ghcr.io/krau/saveany-bot:latest docker restart saveany-bot ``` -docker compose: +### Docker Compose ```bash docker compose pull -docker compose restart -``` \ No newline at end of file +docker compose up -d +``` + +更新前请备份 `data/`,并保留 Docker 数据卷。 diff --git a/docs/content/zh/deployment/migration.md b/docs/content/zh/deployment/migration.md new file mode 100644 index 00000000..780d5eca --- /dev/null +++ b/docs/content/zh/deployment/migration.md @@ -0,0 +1,132 @@ +--- +title: "从 v0 升级" +weight: 15 +--- + +# 从 v0 升级 + +v1 使用本地 SQLite 数据库保存配置。首次启动若检测到 v0 的 `config.toml`,会自动导入数据库;此后所有配置均在 Web 控制台管理。 + +v1 不迁移 Telegram 会话。旧版 `session.db` 与 `usersession.db` 已弃用,迁移仅导入旧版配置并写入新数据库。Bot 将使用现有 Token 重新登录,Userbot 则需在 Web 控制台重新登录。升级前请完整备份数据。 + +## 1. 停止并备份 + +先停止 v0,再备份配置、数据和下载目录: + +```bash +docker compose down +cp -a config.toml data downloads /path/to/backup/ +``` + +如果使用二进制部署,直接停止进程后备份相同目录即可。 + +不要同时运行 v0 和 v1,也不要让两个实例共用同一个 `data/` 目录。 + +## 2. 保留数据目录 + +v1 不复用 v0 数据库。请保留旧文件作为备份,由 v1 创建新的 `data/saveany.db`: + +| v0 文件 | v1 默认路径 | 说明 | +|---|---|---| +| `data/saveany.db` | 不复用 | 保留作为备份;v1 创建新数据库 | + +不要使用 `--db-path` 或 `SAVEANY_DB_PATH` 指向 v0 数据库(旧 schema 不兼容)。请在启动 v1 前将旧的 `data/saveany.db` 移出或重命名备份,由 v1 在默认路径创建新数据库。 + +v0 的 `data/session.db` 和 `data/usersession.db` 会被忽略,确认已备份后可删除。v1 使用新的会话文件: + +| 文件 | 说明 | +|---|---| +| `data/bot_session.db` | Bot 会话 | +| `data/userbot_session.db` | Userbot 会话 | + +Bot 使用当前 Token 重新登录。启用 Userbot 时,请稍后在 Telegram 页面重新登录。 + +## 3. 首次启动时导入配置 + +首次启动必须让 v1 能读到旧版 `config.toml`。Docker 只在这时挂载配置文件: + +```yaml +services: + saveany-bot: + image: ghcr.io/krau/saveany-bot:latest + restart: unless-stopped + ports: + - "28080:28080" + environment: + SAVEANY_LISTEN: 0.0.0.0:28080 + volumes: + - ./data:/app/data + - ./config.toml:/app/config.toml:ro + - ./downloads:/app/downloads + - ./cache:/app/cache +``` + +```bash +docker compose up -d +docker compose logs -f saveany-bot +``` + +二进制部署: + +```bash +./saveany-bot --config /path/to/config.toml +``` + +省略 `--config` 时,v1 会依次查找工作目录下的 `./config.toml` 和 `/etc/saveany/config.toml`。 + +日志中出现配置导入完成后,即可移除 `config.toml` 挂载并重启。v1 在数据库中写入导入标记,不会重复导入。`config.toml` 中的 `db.session` 和 `telegram.userbot.session` 会被忽略,v1 始终使用新的会话文件路径。 + +## 4. 登录并核对 + +启动日志会打印带访问 Token 的 Web UI 直达地址,直接打开即可登录。如果首次启动没有设置 `SAVEANY_ADMIN_PASSWORD`,备用管理员密码也会打印在日志中: + +```bash +docker compose logs saveany-bot | grep -E "Web UI available at|Generated web admin password" +``` + +登录后打开 **指南**,确认: + +- Bot 已使用当前 Token 重新连接。 +- 若启用了 Userbot,已重新登录并显示账号。 +- 存储已启用且没有初始化错误。 +- 用户列表包含原有账号。v0 配置中不存在的旧用户会被禁用,需要时手动启用。 +- 从 `config.toml` 导入的存储、用户和设置仍然存在。 +- 目录、规则和监听需要在 v1 中重新创建;v0 数据库行不会迁移。 + +然后向 Bot 发送一个文件或 Telegram 消息链接,确认任务能正常完成。 + +## 5. 迁移之后 + +- 修改 `config.toml` 不再生效,请直接在 Web 控制台调整设置。 +- MinIO 存储会在启动时转换为 `s3`,请在 Web 控制台核对 Endpoint、密钥和 Bucket。 +- 更新前备份整个 `data/` 目录,并保留 Docker 数据卷。 +- Docker 更新使用 `docker compose pull && docker compose up -d`。 +- 若更换 Bot Token,建议删除 `data/bot_session.db` 后重启,避免旧会话不匹配。 + +## 常见问题 + +### 未导入配置 + +确认配置文件路径正确、内容仍是合法 TOML,并检查启动日志。导入标记只会在导入成功后写入;已有同名存储或配置组不会被覆盖。 + +### 旧会话未迁移 + +v1 不读取旧版 gotgproto 的 `session.db` 或 `usersession.db`。Bot 会使用当前 Token 自动重新登录;启用 Userbot 时,在 Telegram 页面重新登录即可。旧文件可以保留在备份中,确认不再需要后删除。 + +### 旧数据库未迁移 + +v1 不复用 v0 数据库或其 `[db] path`。请保留旧文件作为备份,由 v1 创建新数据库。 + +### 本地存储路径失效 + +v0 的本地存储路径相对于进程工作目录。Docker 中通常是 `/app`,请确认 `downloads/` 仍挂载到同一路径。 + +### 忘记登录密码 + +查看首次启动日志中的 `Generated web admin password`。密码只保存哈希,无法从数据库还原;若日志已丢失,请先备份数据库,再清空其中的 `auth` 设置并重启,重新生成密码。 + +## 相关文档 + +- [安装与更新](../installation) +- [Web 控制台配置](../configuration/) +- [使用帮助](../../usage/) diff --git a/docs/content/zh/help/_index.md b/docs/content/zh/help/_index.md index c1ff08e0..a3ec640a 100644 --- a/docs/content/zh/help/_index.md +++ b/docs/content/zh/help/_index.md @@ -5,14 +5,14 @@ weight: 15 # 常见问题 -## 上传 alist 失败也会显示成功 +## 上传 Alist 失败但界面显示成功 -在 alist 管理页面适当调整上传分片大小, 为 alist 使用更稳定的网络环境部署, 都可以减少这种情况的发生. +可尝试在 Alist 管理页面调小上传分片大小,或改善 Alist 部署环境的网络质量。 -## Bot 提示下载成功但是 alist 未显示 +## Bot 提示下载成功但 Alist 未显示文件 -alist 缓存了目录结构, 参考 文档 可以调整缓存时间 +Alist 缓存了目录结构,可参考 [Alist 缓存文档](https://alist.nn.ci/zh/guide/drivers/common.html#缓存过期) 缩短缓存过期时间或手动刷新目录。 -## docker部署配置了代理后仍无法连接 telegram (初始化客户端超时) +## Docker 部署配置代理后无法连接 Telegram(初始化客户端超时) -docker 不能直接访问宿主机网络, 如果你不熟悉其用法, 请将容器设为 host 模式. \ No newline at end of file +容器默认无法直接通过 `127.0.0.1` 访问宿主机网络。可将代理地址设为宿主机真实局域网 IP,或将容器网络模式改为 `host`。 \ No newline at end of file diff --git a/docs/content/zh/usage/_index.md b/docs/content/zh/usage/_index.md index 93e906cb..459ff77b 100644 --- a/docs/content/zh/usage/_index.md +++ b/docs/content/zh/usage/_index.md @@ -5,12 +5,39 @@ weight: 10 # 使用帮助 -这里介绍 Save Any Bot 的一些功能和使用方法, 如果你没有在这里找到你需要的内容, 另请参阅 [配置说明](../deployment/configuration) 或前往 Github [Discussions](https://github.com/krau/SaveAny-Bot/discussions) 提问. +介绍 Save Any Bot 的常用功能与命令操作。如需部署及高级配置,请参阅[配置说明](../deployment/configuration);其他问题可在 GitHub [Discussions](https://github.com/krau/SaveAny-Bot/discussions) 交流。 + +## Bot 命令 + +| 命令 | 用途 | +|---|---| +| `/start` | 打开帮助和命令列表 | +| `/save` | 保存回复的媒体消息,或按消息范围批量保存 | +| `/download` | 使用直接下载、Aria2 或视频解析下载链接 | +| `/transfer` | 在存储端之间传输文件 | +| `/tasks` | 查看和管理运行中、排队中和失败任务 | +| `/config` | 用户设置:文件名和重名策略、静默模式、默认存储 | +| `/template` | 设置自定义文件名模板 | +| `/dirs` | 管理存储目录 | +| `/rules` | 管理存储规则 | +| `/watch` | 监听聊天并管理通知/对话同步 (UserBot) | +| `/parser` | 管理解析器插件 | +| `/update` | 检查更新 | ## 转存文件 -要使用 Bot 的转存 Telegram 文件功能, 需要向 Bot 发送或转发以下类型的消息. +向 Bot 发送或转发以下内容即可触发转存: + +1. 图片、视频、音频、文档等媒体文件 +2. Telegram 消息链接(如 `https://t.me/acherkrau/1097`),支持转存开启了防转发/防保存保护的频道文件 +3. Telegra.ph 文章链接,自动提取并下载其中的所有图片 + +## 在群组中使用 + +Bot 支持普通群组和超级群组,不支持广播频道。 -1. 文件或媒体消息, 如图片, 视频, 文档等 -2. Telegram 消息链接, 例如: `https://t.me/acherkrau/1097`. **即使频道禁止了转发和保存, Bot 依然可以下载其文件.** -3. Telegra.ph 的文章链接, Bot 将下载其中的所有图片 \ No newline at end of file +- 群组中仅响应显式命令和按钮回调,不会自动保存群内的普通媒体或普通文本消息。 +- 转存群内文件时,回复目标消息并发送 `/save`(群组中亦可使用 `/save@YourBot` 格式);其他命令(如 `/download`)用法相同。 +- 仅白名单用户可以使用。非白名单成员的消息会被忽略,点击内联按钮将提示无权限。 +- 超级群中以匿名管理员身份发送的消息因无法识别具体用户,会被直接忽略。 +- 默认的 Privacy Mode 下,Bot 仅能接收针对它的命令、@提及以及对其消息的回复。若需接收群内全部消息,可在 [@BotFather](https://t.me/BotFather) 使用 `/setprivacy` 关闭 Privacy Mode;Bot 在群组中依然仅响应命令和交互按钮。 \ No newline at end of file diff --git a/docs/content/zh/usage/api.md b/docs/content/zh/usage/api.md index 3a9cfb4e..fdda8c46 100644 --- a/docs/content/zh/usage/api.md +++ b/docs/content/zh/usage/api.md @@ -5,45 +5,33 @@ weight: 20 # HTTP API -SaveAny-Bot 提供了一套 HTTP API,允许你通过程序化方式创建下载/转存任务、查询任务状态、取消任务等,无需通过 Telegram 操作。 +SaveAny-Bot 提供 HTTP API,支持通过脚本或第三方程序创建下载/转存任务、查询状态及取消任务。 ## 启用 API -在 `config.toml` 中添加或修改以下配置: +API 与 Web 控制台共用监听地址和端口,不需要单独启用。Web 控制台默认监听 `127.0.0.1:28080`。在 **设置 → Web** 中配置访问 Token: -```toml -[api] -enable = true -host = "0.0.0.0" # 监听地址,默认 0.0.0.0 -port = 8080 # 监听端口,默认 8080 -token = "your-token" # 鉴权 Token,强烈建议设置 -``` - -也可通过环境变量覆盖(前缀 `SAVEANY_`): +- `访问 Token`:用于 `/?token=...` 直接登录 Web UI,也用于 API 的 `Authorization: Bearer `。 +- `监听地址`:控制台与 API 共用;需要从其他主机访问时使用 `0.0.0.0:28080`。 -| 环境变量 | 对应配置项 | -|---|---| -| `SAVEANY_API_ENABLE` | `api.enable` | -| `SAVEANY_API_HOST` | `api.host` | -| `SAVEANY_API_PORT` | `api.port` | -| `SAVEANY_API_TOKEN` | `api.token` | +所有接口位于 `/api/v1` 下。 -{{< hint warning >}} -若 `token` 为空,API 服务将**不进行任何鉴权**即可访问,存在安全风险。 -{{< /hint >}} +## 认证 -## 鉴权 - -当配置了 `token` 时,所有 API 请求均需在 HTTP 请求头中携带 Bearer Token: +在 **设置 → Web** 中配置访问 Token,然后在请求头中携带: ``` Authorization: Bearer ``` +未配置访问 Token 时,每次启动都会生成随机 Token,并作为 Web UI 直达地址的一部分打印到日志。该 Token 在当前进程内同样可用于 Bearer 认证。 + +也可以使用 Web 控制台登录 Cookie。使用 Cookie 的非 GET 请求还需要携带 `X-CSRF-Token`。 + 鉴权失败时返回 `401`: ```json -{ "error": "unauthorized", "message": "invalid token" } +{ "error": "unauthorized", "message": "authentication required" } ``` ## 错误响应格式 @@ -62,7 +50,6 @@ Authorization: Bearer | 错误码 | HTTP 状态 | 含义 | |---|---|---| | `unauthorized` | 401 | 鉴权失败 | -| `method_not_allowed` | 405 | HTTP 方法不正确 | | `invalid_request` | 400 | 请求体/参数非法 | | `task_creation_failed` | 400 | 任务创建失败 | | `task_not_found` | 404 | 任务 ID 不存在 | @@ -80,7 +67,7 @@ Authorization: Bearer **响应 `200 OK`:** ```json -{ "status": "ok" } +{ "status": "ok", "version": "1.0.0" } ``` --- @@ -94,14 +81,44 @@ Authorization: Bearer ```json { "storages": [ - { "name": "local", "type": "local" }, - { "name": "MyMinio", "type": "s3" } + { + "storage_id": 1, + "name": "local", + "type": "local", + "enable": true, + "config": { "base_path": "/app/downloads" }, + "config_version": 1 + } ] } ``` --- +### GET /api/v1/meta — API 与能力元数据 + +返回冻结的 v1 能力词表和数据库 schema 版本。 + +```json +{ + "api_version": "v1", + "version": "1.0.0", + "schema_version": 1, + "chat_id_format": "bot_api", + "task_types": ["tgfiles", "tphpics", "parseditem", "directlinks", "aria2", "ytdlp", "transfer"], + "task_statuses": ["queued", "running", "completed", "partial", "failed", "cancelled"], + "task_item_statuses": ["completed", "failed", "skipped", "cancelled"], + "storage_types": ["local", "webdav", "alist", "telegram", "s3", "rclone"], + "rule_types": ["FILENAME-REGEX", "MESSAGE-REGEX", "IS-ALBUM"], + "filename_strategies": ["default", "message", "template"], + "conflict_strategies": ["rename", "ask", "overwrite", "skip"] +} +``` + +聊天 ID 使用 Bot API 形式:用户为正数,普通群为负数,超级群和频道使用 `-100...`。 + +--- + ### GET /api/v1/task-types — 列出支持的任务类型 **响应 `200 OK`:** @@ -109,13 +126,8 @@ Authorization: Bearer ```json { "types": [ - "directlinks", - "ytdlp", - "aria2", - "parseditem", - "tgfiles", - "tphpics", - "transfer" + { "type": "directlinks", "label": "directlinks" }, + { "type": "tgfiles", "label": "tgfiles" } ] } ``` @@ -124,12 +136,12 @@ Authorization: Bearer ### GET /api/v1/media-metadata — 媒体元数据查询 -对 Telegram 消息链接、直链音视频地址或其他 yt-dlp 支持的媒体链接进行探测,在不完整下载文件的情况下返回标题、缩略图、上传者和音视频时长等可用元数据。 +探测 Telegram 消息链接、直链音视频或 yt-dlp 支持的媒体链接,无需完整下载即可提取标题、缩略图、上传者和时长等元数据。 请求: ```bash -curl -X GET "http://localhost:8080/api/v1/media-metadata?url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dxxx" \ +curl -X GET "http://127.0.0.1:28080/api/v1/media-metadata?url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3Dxxx" \ -H "Authorization: Bearer YOUR_TOKEN" ``` @@ -163,7 +175,7 @@ Authorization: Bearer ```json { "type": "<任务类型>", - "storage": "<存储名>", + "storage_id": 1, "path": "<子目录>", "webhook": "<回调URL>", "params": { } @@ -173,7 +185,7 @@ Authorization: Bearer | 字段 | 类型 | 必填 | 说明 | |---|---|---|---| | `type` | string | 是 | 任务类型,见下文 | -| `storage` | string | 是 | 目标存储名,须与配置中的存储名一致 | +| `storage_id` | uint | 是 | 目标存储 ID | | `path` | string | 否 | 存储内的子目录路径 | | `webhook` | string | 否 | 任务完成/失败时的回调地址 | | `params` | object | 是 | 各任务类型的专属参数,见下文 | @@ -182,9 +194,11 @@ Authorization: Bearer ```json { - "task_id": "abc123xyz", - "type": "directlinks", - "status": "queued", + "task_id": "abc123xyz", + "type": "directlinks", + "status": "queued", + "storage_id": 1, + "storage_name": "local", "created_at": "2026-03-11T10:00:00Z" } ``` @@ -198,7 +212,7 @@ Authorization: Bearer ```json { "type": "directlinks", - "storage": "local", + "storage_id": 1, "path": "downloads", "params": { "urls": [ @@ -219,12 +233,12 @@ Authorization: Bearer 需要在系统中安装 yt-dlp。 {{< /hint >}} -通过 yt-dlp 下载视频/音频,支持 YouTube、Bilibili 等 1000+ 网站。 +通过 yt-dlp 下载音视频,支持主流视频网站。 ```json { "type": "ytdlp", - "storage": "local", + "storage_id": 1, "path": "videos", "params": { "urls": ["https://www.youtube.com/watch?v=xxx"], @@ -241,7 +255,7 @@ Authorization: Bearer ##### aria2 — Aria2 下载 {{< hint warning >}} -需要在配置文件中启用并配置 Aria2 RPC。 +需要在 **设置 → Aria2** 中启用并配置 Aria2 RPC。 {{< /hint >}} 通过 Aria2 下载管理器下载文件,支持 HTTP/HTTPS、FTP、BitTorrent(磁力链接、种子)等协议。 @@ -249,7 +263,7 @@ Authorization: Bearer ```json { "type": "aria2", - "storage": "local", + "storage_id": 1, "path": "downloads", "params": { "urls": ["magnet:?xt=urn:btih:..."], @@ -270,7 +284,7 @@ Authorization: Bearer ```json { "type": "parseditem", - "storage": "local", + "storage_id": 1, "path": "parsed", "params": { "url": "https://some-site.com/page" @@ -298,7 +312,7 @@ Authorization: Bearer ```json { "type": "tgfiles", - "storage": "local", + "storage_id": 1, "path": "telegram", "params": { "message_links": [ @@ -311,7 +325,10 @@ Authorization: Bearer | params 字段 | 类型 | 必填 | 说明 | |---|---|---|---| -| `message_links` | []string | 是 | Telegram 消息链接列表,至少 1 条 | +| `message_links` | []string | 否 | Telegram 消息链接列表;与 `chat_id` + `message_range` 二选一 | +| `chat_id` | string | 否 | 与消息范围配合使用的 Bot API 聊天 ID 或用户名 | +| `message_range` | string | 否 | 消息范围,例如 `100-120` | +| `filter` | string | 否 | 可选正则,匹配消息文本或文件名 | ##### tphpics — Telegraph 文章图片下载 @@ -322,7 +339,7 @@ Authorization: Bearer ```json { "type": "tphpics", - "storage": "local", + "storage_id": 1, "path": "telegraph", "params": { "telegraph_url": "https://telegra.ph/Some-Article-01-01" @@ -339,34 +356,34 @@ Authorization: Bearer 在两个存储后端之间直接传输文件,无需经过 Telegram。源存储须支持列举(list)和读取(read)操作。 {{< hint info >}} -`transfer` 任务中,顶层的 `storage` 字段仍然必须填写(用于通过参数校验),但实际使用的存储由 `params` 中的 `source_storage` 和 `target_storage` 决定。 +`transfer` 任务中,顶层的 `storage_id` 字段仍然必须填写(用于通过参数校验),但实际使用的存储由 `params` 中的 `source_storage_id` 和 `target_storage_id` 决定。 {{< /hint >}} ```json { "type": "transfer", - "storage": "local", + "storage_id": 1, "params": { - "source_storage": "MyS3", - "source_path": "backups/", - "target_storage": "LocalDisk", - "target_path": "restored/" + "source_storage_id": 2, + "source_path": "backups/", + "target_storage_id": 1, + "target_path": "restored/" } } ``` | params 字段 | 类型 | 必填 | 说明 | |---|---|---|---| -| `source_storage` | string | 是 | 源存储名 | +| `source_storage_id` | uint | 是 | 源存储 ID | | `source_path` | string | 是 | 源存储中的路径,须包含至少一个文件 | -| `target_storage` | string | 是 | 目标存储名 | +| `target_storage_id` | uint | 是 | 目标存储 ID | | `target_path` | string | 是 | 目标存储中的路径 | --- ### GET /api/v1/tasks — 列出所有任务 -返回所有 API 创建的任务,按创建时间由新到旧排序(仅在内存中保留,重启后清空)。已进入终态的任务在最后一次更新 24 小时后也会被清理,清理任务每 10 分钟执行一次。 +返回所有任务,按创建时间由新到旧排序。任务记录和创建参数持久化在 SQLite 中;排队中和中断的任务会在重启后恢复。终态记录保留到通过 API 删除。 **响应 `200 OK`:** @@ -378,7 +395,8 @@ Authorization: Bearer "type": "directlinks", "status": "running", "title": "file.zip", - "storage": "local", + "storage_id": 1, + "storage_name": "local", "path": "downloads", "error": "", "created_at": "2026-03-11T10:00:00Z", @@ -411,25 +429,113 @@ Authorization: Bearer - `400 invalid_request` — 路径中未提供 task_id - `404 task_not_found` — 任务不存在 +任务对象在至少一个文件保存了消息元数据时会返回 `metadata_available`。 + --- -### DELETE /api/v1/tasks/{task_id} — 取消任务 +### GET /api/v1/tasks/{task_id}/metadata — 查询消息元数据 -**路径参数:** `task_id` +返回 Telegram 文件任务中每个已保存文件的消息元数据。需要先在 **设置 → 消息元数据** 中启用;元数据始终保存在本地数据库。启用“写入旁车 JSON”时还会在实际文件旁写入 `<文件路径>.meta.json`,Telegram 存储会跳过旁车。 + +**路径参数:** `task_id` — 要查询的任务。 **响应 `200 OK`:** ```json -{ "message": "task cancelled successfully" } +{ + "task_id": "abc123xyz", + "items": [ + { + "index": 1, + "name": "saved-video.mp4", + "path": "downloads/saved-video.mp4", + "metadata": { + "version": 1, + "message": { + "id": 123, + "chat": { "id": -1001234567890, "title": "Channel" }, + "sender": { "id": 123456789, "name": "Alice" }, + "text": "caption", + "grouped_id": 999 + }, + "file": { + "name": "saved-video.mp4", + "media_type": "document", + "mime_type": "video/mp4", + "size": 12345678, + "width": 1920, + "height": 1080, + "duration": 123.4 + } + } + } + ] +} ``` +关闭“保存消息文本”时不会保存消息正文。不会保存 access hash、Token 或原始 Telegram 对象。 + **错误响应:** - `400 invalid_request` — 路径中未提供 task_id - `404 task_not_found` — 任务不存在 +- `404 metadata_not_found` — 该任务没有已记录的元数据 + +--- + +### POST /api/v1/tasks/{task_id}/retry — 重试失败文件 + +使用已完成任务中失败的文件创建新任务。Bot 会重新拉取原始消息,同时刷新文件引用。仅支持 Telegram 文件任务(`tgfiles`)。 + +**路径参数:** `task_id` — 要重试的任务。 + +**响应 `202 Accepted`:** + +```json +{ "task_id": "d4f7u1crvimcs4n4j8e1", "status": "queued", "file_count": 2 } +``` + +**错误响应:** +- `409 task_active` — 任务仍在排队或运行中 +- `409 nothing_to_retry` — 没有失败或未完成的文件 +- `409 storage_missing` / `409 source_unavailable` +- `400 retry_unsupported` — 该任务类型不支持重试 +- `503 client_unavailable` — 没有运行中的 Telegram 客户端 + +任务响应包含单文件结果:列表响应返回失败总数(`failed_count`)以及每个任务最多 5 个失败项(`failed_items`),单任务查询返回全部记录(`items`)。 + +--- + +### POST /api/v1/tasks/{task_id}/cancel — 取消任务 + +取消排队中或运行中的任务。 + +**路径参数:** `task_id` — 要取消的任务。 + +**响应 `202 Accepted`:** + +```json +{ "task_id": "abc123xyz", "status": "cancelled" } +``` + +**错误响应:** +- `409 task_not_active` — 任务已经结束 +- `404 task_not_found` — 任务不存在 - `500 cancel_failed` — 取消操作失败 --- +### DELETE /api/v1/tasks/{task_id} — 删除任务 + +删除已结束的任务及其记录。取消排队中或运行中的任务请使用 `POST /api/v1/tasks/{task_id}/cancel`。 + +**路径参数:** `task_id` + +**响应 `204 No Content`:** +- `409 task_active` — 请先取消任务 +- `404 task_not_found` — 任务不存在 + +--- + ## 任务状态 | 状态值 | 含义 | @@ -437,17 +543,20 @@ Authorization: Bearer | `queued` | 已入队,等待执行 | | `running` | 正在执行 | | `completed` | 已成功完成 | +| `partial` | 已完成,但部分文件失败 | | `failed` | 执行失败 | -| `cancelled` | 已通过 DELETE 接口取消 | +| `cancelled` | 已通过取消接口取消 | + +排队中的任务和进程停止时正在运行的任务都会保存创建参数,重启后自动重新入队。Telegram 下载会从部分缓存继续,已完成上传的文件会被跳过。 --- ## Webhook 回调 -创建任务时可设置 `webhook` 字段。当任务进入 `completed` 或 `failed` 状态时,Bot 会向该地址发送一个 `POST` 请求。 +创建任务时可设置 `webhook` 字段。当任务进入 `completed`、`partial` 或 `failed` 状态时,Bot 会向该地址发送一个 `POST` 请求。 {{< hint info >}} -被取消的任务**不会**触发 Webhook——取消操作的发起方已经知晓结果。每个任务最多只会发送一次回调。 +已取消的任务不会触发 Webhook,每个任务最多发送一次回调。 {{< /hint >}} 回调为异步发送,不会因为回调地址响应缓慢或不可达而阻塞任务。 @@ -466,13 +575,14 @@ User-Agent: SaveAny-Bot/1.0 "task_id": "abc123xyz", "type": "directlinks", "status": "completed", - "storage": "local", + "storage_id": 1, + "storage_name": "local", "path": "downloads", "completed_at": "2026-03-11T10:01:00Z", "error": "" } ``` -`completed_at` 仅在状态为 `completed` 或 `failed` 时出现。`error` 仅在有错误时出现。 +`completed_at` 仅在状态为 `completed`、`partial` 或 `failed` 时出现。`error` 仅在有错误时出现。 **重试机制:** 最多尝试 3 次。响应非 2xx 或请求出错时会重试,间隔依次为 100 毫秒、400 毫秒。每次请求超时为 30 秒。 diff --git a/docs/content/zh/usage/aria2.md b/docs/content/zh/usage/aria2.md index d4cd9e58..33723360 100644 --- a/docs/content/zh/usage/aria2.md +++ b/docs/content/zh/usage/aria2.md @@ -6,36 +6,32 @@ weight: 6 # Aria2 下载 {{< hint warning >}} -该功能需要在配置文件中启用 Aria2 并配置 RPC 连接. +需要在 **设置 → Aria2** 中启用 Aria2 并配置 RPC 连接。 {{< /hint >}} -使用 `/aria2dl` 命令可以通过 Aria2 下载管理器下载文件, 支持 HTTP/HTTPS、FTP、BitTorrent 等多种协议. +可通过 `/download` 使用 Aria2 下载,支持 HTTP/HTTPS、FTP、BitTorrent 种子与磁力链接。发送链接后选择 **Aria2** 即可。 ```bash -/aria2dl [uri2] [uri3] ... +/download <链接1> [链接2] [链接3] ... ``` -示例: +示例: ```bash # 下载 HTTP 链接 -/aria2dl https://example.com/file.zip +/download https://example.com/file.zip # 下载磁力链接 -/aria2dl magnet:?xt=urn:btih:... +/download magnet:?xt=urn:btih:... -# 下载种子文件 (需要先上传 .torrent 文件) -/aria2dl https://example.com/file.torrent +# 下载种子文件 +/download https://example.com/file.torrent ``` -配置 Aria2: +选择 **Aria2** 后指定目标存储位置。若链接仅 Aria2 支持,将直接进入 Aria2 下载流程。 -在 `config.toml` 中添加: +在 Web 控制台的 **设置 → Aria2** 中配置: -```toml -[aria2] -enable = true -url = "http://localhost:6800/jsonrpc" -secret = "your-rpc-secret" # 如果配置了 rpc-secret -remove_after_transfer = true # 转存完成后删除本地文件 -``` +- `URL`: Aria2 RPC 地址,例如 `http://localhost:6800/jsonrpc` +- `Secret`: 配置了 `rpc-secret` 时填写 +- `保留文件`: 转存完成后保留 Aria2 下载的本地文件 diff --git a/docs/content/zh/usage/cli.md b/docs/content/zh/usage/cli.md index 05b925d2..2b4a5361 100644 --- a/docs/content/zh/usage/cli.md +++ b/docs/content/zh/usage/cli.md @@ -5,9 +5,9 @@ weight: 21 # 命令行子命令 -除了直接运行 `./saveany-bot` (不带子命令) 启动 Telegram Bot 外, 这个二进制文件还提供两个把本地文件上传到存储后端的辅助子命令: `upload` (一次性) 和 `watch` (持续监听). +除直接运行 `./saveany-bot` 启动 Telegram Bot 外,二进制文件还提供两个辅助子命令用于上传本地文件:`upload`(一次性上传)和 `watch`(持续监听目录并上传)。 -这些子命令会读取与 Bot 相同的 `config.toml`, 初始化数据库和缓存, 然后执行任务. 它们**不会**启动 Telegram Bot 本身, 但 `telegram` 类型的存储会在需要上传时临时启动 Bot 客户端来执行上传. +子命令读取相同的数据库配置与缓存,但不会启动 Bot 服务。当目标存储为 `telegram` 时,会在上传期间临时启动 Bot 客户端。 ## `upload` — 上传单个文件 @@ -20,7 +20,7 @@ saveany-bot upload -f <文件> -s <存储名> [-d <目录>] [--no-progress] | 参数 | 必填 | 说明 | |---|---|---| | `-f, --file` | 是 | 待上传的本地文件路径 | -| `-s, --storage` | 是 | 目标存储名 (必须存在于 `config.toml`) | +| `-s, --storage` | 是 | 目标存储名 (必须在 Web 控制台的存储页面中存在) | | `-d, --dir` | 否 | 存储中的目标目录, 默认使用存储的 `base_path` | | `--no-progress` | 否 | 关闭终端进度条 | @@ -82,9 +82,9 @@ saveany-bot watch -p <路径> -s <存储名> [-d <目录>] [选项] - 相对子目录结构会被保留: 以 `--path ./inbox` 为例, 写入 `./inbox/sub/file.txt` 的文件会被上传到 `<目标目录>/sub/file.txt`. - `watch` 会一直运行直到被中断 (如 `Ctrl-C` / `SIGINT`), 退出前会等待所有进行中的上传完成. -- 重试次数遵循 `config.toml` 中的全局 `retry` 值, 各次重试之间间隔 `--retry-delay`. +- 重试次数遵循 设置中的全局 `retry` 值, 各次重试之间间隔 `--retry-delay`. - `telegram` 类型的存储会自动启动 Bot 客户端来执行上传. {{< hint warning >}} -`watch` 子命令与 Bot 内的 `/watch` 命令 (监听 Telegram 聊天) 无关. 本子命令监听的是**本地文件系统目录**, 不依赖 Telegram. +`watch` 子命令用于监听本地文件目录,与 Telegram 聊天的 `/watch` 命令无关。 {{< /hint >}} \ No newline at end of file diff --git a/docs/content/zh/usage/config.md b/docs/content/zh/usage/config.md index de8996b5..62dde828 100644 --- a/docs/content/zh/usage/config.md +++ b/docs/content/zh/usage/config.md @@ -5,47 +5,50 @@ weight: 11 # 文件命名与重名策略 -SaveAny-Bot 支持在 Telegram 中通过 `/config` 和 `/fnametmpl` 命令自定义保存文件的命名方式, 以及处理与已存在文件重名时的冲突策略. +可通过 `/config` 和 `/template` 自定义文件命名方式与重名冲突策略。 ## `/config` — 用户配置 -`/config` 命令会弹出一个内联菜单, 你可以在其中修改以下两项用户级设置: +`/config` 命令包含以下设置: -- **文件名策略** — 保存文件的命名方式 -- **重名文件保存策略** — 目标存储中已存在同名文件时的处理方式 +- **文件名策略**:保存文件的命名方式 +- **重名文件保存策略**:目标存储中已存在同名文件时的处理方式 +- **静默模式**:直接保存到默认位置,不再每次询问 +- **默认存储**:选择静默模式和无规则匹配时使用的默认存储及目录 +- **文件名模板**:查看当前模板与 `/template` 用法 -设置按用户分别保存, 对该用户后续所有的保存/转存任务生效. +设置按用户分别保存,对该用户后续所有的保存与转存任务生效。 ### 文件名策略 | 选项 | 行为 | |---|---| -| `默认` | 使用媒体原始文件名, 没有原始文件名时使用生成名 | -| `优先从消息生成` | 优先根据消息内容 (如 caption、文本) 生成文件名, 而非原始文件名 | -| `自定义模板` | 使用 `/fnametmpl` 设置的自定义模板渲染文件名 | +| `默认` | 使用媒体原始文件名,无原始文件名时使用生成名称 | +| `优先从消息生成` | 优先根据消息内容(如 caption、文本)生成文件名,而非原始文件名 | +| `自定义模板` | 使用 `/template` 设置的自定义模板渲染文件名 | ### 重名文件保存策略 | 选项 | 行为 | |---|---| -| `始终重命名` (默认) | 保留已有文件, 将新文件以另一个名字保存 | -| `每次询问` | 每次遇到重名时通过内联按钮提示你选择 | +| `始终重命名` (默认) | 保留已有文件,新文件自动追加后缀保存 | +| `每次询问` | 每次遇到重名时通过内联按钮提示选择 | | `始终覆盖` | 用新文件替换已有文件 | -| `始终跳过` | 对重名文件不做处理 | +| `始终跳过` | 跳过不保存 | {{< hint info >}} 重名策略仅在能够检测文件是否已存在的存储后端生效. 不支持检测文件是否存在的存储后端会退化为覆盖行为. {{< /hint >}} -## `/fnametmpl` — 自定义文件名模板 +## `/template` — 自定义文件名模板 -当文件名策略设置为 `自定义模板` 时, SaveAny-Bot 会用 `/fnametmpl` 配置的模板来渲染所保存文件的文件名. +当文件名策略设置为 `自定义模板` 时, SaveAny-Bot 会用 `/template` 配置的模板来渲染所保存文件的文件名. ``` -/fnametmpl [模板] +/template [模板] ``` -- 不带参数运行 `/fnametmpl` 会显示当前模板以及帮助说明 +- 不带参数运行 `/template` 会显示当前模板以及帮助说明 - 带模板字符串运行则会把它设为你的文件名模板 模板使用 Go [`text/template`](https://pkg.go.dev/text/template) 语法. 可用变量如下: @@ -64,12 +67,16 @@ SaveAny-Bot 支持在 Telegram 中通过 `/config` 和 `/fnametmpl` 命令自定 ``` # 固定前缀 + 消息 ID + 日期 -/fnametmpl 图片_{{.msgid}}_{{.msgdate}}.jpg +/template 图片_{{.msgid}}_{{.msgdate}}.jpg # 优先使用原始文件名, 没有则用生成名 -/fnametmpl {{.origname}} +/template {{.origname}} ``` +{{< hint info >}} +当媒体组按批次保存时 (发送给 Bot 的媒体组, 以及命中 `NEW-FOR-ALBUM` 规则的监听媒体组), 没有自身文本的文件会使用媒体组的文本填充模板变量和消息规则, 因此每个文件都能取得 `{{.msgtags}}` 与 `{{.msgraw}}`. 若模板对多个文件产生相同名称, 后续文件会保留自动生成的名称. +{{< /hint >}} + {{< hint warning >}} 模板仅在文件名策略设置为 `自定义模板` 时生效. 如果模板解析失败, SaveAny-Bot 会回退到默认的文件名生成逻辑. {{< /hint >}} \ No newline at end of file diff --git a/docs/content/zh/usage/directlinks.md b/docs/content/zh/usage/directlinks.md index dd41f682..532b0ab7 100644 --- a/docs/content/zh/usage/directlinks.md +++ b/docs/content/zh/usage/directlinks.md @@ -1,21 +1,25 @@ --- -title: "直接下载链接" +title: "链接下载" weight: 5 --- -# 直接下载链接 +# 链接下载 -使用 `/dl` 命令可以直接下载一个或多个 HTTP/HTTPS 链接的文件到存储中. +使用 `/download` 命令发送一个或多个链接,并选择下载方式: + +- **直接下载** — 普通 HTTP/HTTPS 文件链接 +- **Aria2** — 启用 Aria2 后可用,也支持磁力和种子链接 +- **视频** — 安装 yt-dlp 后可用,用于支持的视频网站 ```bash -/dl [url2] [url3] ... +/download <链接1> [链接2] [链接3] ... ``` -示例: +示例: ```bash -/dl https://example.com/file.zip -/dl https://example.com/file1.zip https://example.com/file2.zip +/download https://example.com/file.zip +/download https://example.com/file1.zip https://example.com/file2.zip ``` -Bot 会验证链接格式, 然后让你选择目标存储位置. +选择下载方式后,再选择目标存储位置。视频链接可以在同一命令后追加 yt-dlp 参数,例如 `-f best` 或 `--extract-audio`。 diff --git a/docs/content/zh/usage/parsers.md b/docs/content/zh/usage/parsers.md index a44d65b1..3a1a27f3 100644 --- a/docs/content/zh/usage/parsers.md +++ b/docs/content/zh/usage/parsers.md @@ -5,11 +5,38 @@ weight: 9 # 转存 Telegram 之外的文件 -除了 Telegram 上的文件, Bot 还可通过 JavaScript 插件或内置解析器来支持转存其他网站的文件. +除 Telegram 文件外,Bot 还可通过内置解析器或 JavaScript 插件转存第三方网站内容。 -> 查看[贡献解析器](../contribute)文档了解详情 +> 编写与贡献解析器请参阅[相关文档](../contribute)。 -只需向 Bot 发送符合解析器要求的链接即可使用, 当前内置的解析器: +向 Bot 发送符合解析器要求的链接。内置解析器: - Twitter - Kemono + +## JavaScript 插件 API + +JavaScript 插件通过以下结构注册解析器: + +```js +registerParser({ + metadata: { + name: 'example', + version: '1.0.0', + apiVersion: '1', + description: '示例插件', + author: 'you', + }, + canHandle(url) { return url.includes('example.com') }, + parse(url) { + return { + site: 'example', + url, + title: 'Example', + resources: [{ url: 'https://cdn.example.com/file.zip', filename: 'file.zip' }], + } + }, +}) +``` + +`apiVersion` 是插件 ABI 版本,与插件自身的 `version` 独立。当前 ABI 为 `"1"`,其他值会被拒绝。可用全局对象为 `console`、`ghttp`(`get`、`getJSON`、`head`)和 `playwright`。`parse` 返回 `Item`,其中 `resources` 支持 `url`、`filename`,以及可选的 `mime_type`、`extension`、`size`、`hash`、`headers`、`extra`。 diff --git a/docs/content/zh/usage/rules.md b/docs/content/zh/usage/rules.md index 4ce6ace4..c5862134 100644 --- a/docs/content/zh/usage/rules.md +++ b/docs/content/zh/usage/rules.md @@ -5,51 +5,54 @@ weight: 3 # 存储规则 -允许你为 Bot 在上传文件到存储时设置一些重定向规则, 用于自动整理所保存的文件. +通过设置重定向规则,自动按文件名、消息内容或相册将保存的文件分类归档。 见: #28 +发送 `/rules` 可查看现有规则、开关规则匹配或删除规则。添加规则与导入预设命令如下: + 目前支持的规则类型: 1. FILENAME-REGEX 2. MESSAGE-REGEX 3. IS-ALBUM -添加规则的基本语法: +添加规则语法: -"规则类型 规则内容 存储名 路径" +`/rules add <规则类型> <匹配内容> <存储ID|存储名> <路径>` -注意空格的使用, 语法正确 bot 才能解析, 以下是一条合法的添加规则命令: +存储参数为纯数字时按存储 ID 解析(可在 Web 控制台的存储页面查看),否则按存储名解析。存储名包含空格时请使用 ID 或为名称加双引号。示例: ``` -/rule add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ MyAlist /视频 +/rules add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ 3 /视频 +/rules add FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ "我的 Alist" /视频 ``` -此外, 规则中的存储名若使用 "CHOSEN" , 则表示存储到点击按钮选择的存储端的路径下 +若存储填为 `CHOSEN`,则表示保存到所选存储端的指定路径下。 -你也可以使用 `/rule switch` 来开关规则模式. 关闭规则模式时, 所有文件都将保存到默认存储. +可通过 `/rules switch` 开关规则模式。关闭时所有文件直接存入默认存储。 ## 预设规则 -为常见文件类型手动编写正则规则比较繁琐, 因此 Bot 内置了一组预设分类 (视频、图片、音频、文档、压缩包), 可以通过一条命令批量导入: +Bot 内置了常见文件类型(视频、图片、音频、文档、压缩包)的分类规则,可一键导入: ``` -/rule preset <存储名> [基础路径] +/rules preset <存储ID|存储名> [基础路径] ``` 参数: -- `存储名`: 目标存储名 (必须存在且你有权访问) +- `存储`: 目标存储 ID 或名称 (必须存在且你有权访问) - `基础路径`: 可选. 各预设分类的子目录会创建在此路径下; 若不填则直接使用默认分类目录名 示例: ``` # 导入预设规则到 "MyAlist", 使用默认目录布局 -/rule preset MyAlist +/rules preset MyAlist # 在自定义基础路径 "downloads/sorted" 下导入预设规则 -/rule preset MyAlist downloads/sorted +/rules preset MyAlist downloads/sorted ``` 此命令会为每个分类创建 `FILENAME-REGEX` 规则, 将匹配的文件路由到 `基础路径` 下对应的子目录: @@ -63,7 +66,7 @@ weight: 3 | 压缩包 | zip, rar, 7z, tar, gz, bz2, xz, ... | `压缩包` | {{< hint info >}} -导入后的预设规则就是普通的 `FILENAME-REGEX` 规则. 你可以像其他规则一样通过 `/rule` 查看或用 `/rule del ` 单独删除/编辑它们. +导入后的预设规则就是普通的 `FILENAME-REGEX` 规则. 你可以像其他规则一样通过 `/rules` 查看或单独删除它们. {{< /hint >}} 规则类型: @@ -76,17 +79,17 @@ weight: 3 FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ MyAlist /视频 ``` -表示将文件名后缀为 mp4,mkv,ts,avi,flv 的文件放到名为 MyAlist 存储下的 /视频 目录内 (同时受配置文件中的 `base_path` 影响) +表示将文件名后缀为 mp4,mkv,ts,avi,flv 的文件放到名为 MyAlist 存储下的 /视频 目录内 (同时受存储设置中的 `base_path` 影响) ## MESSAGE-REGEX -同上, 但是是根据消息本身的文本内容正则匹配 +根据消息正文或说明文字(caption)进行正则表达式匹配。 ## IS-ALBUM -匹配相册消息 (media group), 规则内容只能为 `true` 或 `false`. +匹配相册消息(media group),规则内容为 `true` 或 `false`。 -规则中的路径若使用 "NEW-FOR-ALBUM" , 则表示为该组消息新建一个文件夹来存储它们. 见: https://github.com/krau/SaveAny-Bot/issues/87 +若路径指定为 `NEW-FOR-ALBUM`,则会为整组消息自动新建文件夹并统一保存(参考 issue [#87](https://github.com/krau/SaveAny-Bot/issues/87))。 例如: @@ -94,4 +97,4 @@ FILENAME-REGEX (?i)\.(mp4|mkv|ts|avi|flv)$ MyAlist /视频 IS-ALBUM true MyWebdav NEW-FOR-ALBUM ``` -这将会把以 media group 形式发送的消息保存到名为 MyWebdav 的存储下, 并为每个相册新建一个文件夹(由第一个文件生成)来存储它们. +这将把媒体组消息保存到 MyWebdav 存储下,并为每个相册以首个文件的名称新建子目录归档。 diff --git a/docs/content/zh/usage/silent.md b/docs/content/zh/usage/silent.md index 4361b707..5acdbfe7 100644 --- a/docs/content/zh/usage/silent.md +++ b/docs/content/zh/usage/silent.md @@ -3,12 +3,11 @@ title: "静默模式" weight: 2 --- -# 静默模式 (silent) +# 静默模式 -使用 `/silent` 命令可以开关静默模式. +静默模式可在 `/config` 中开启或关闭。 -默认情况下不开启静默模式, Bot 会询问你每个文件的保存位置. +- **关闭(默认)**:Bot 接收文件时会弹出选择菜单,每次询问保存位置。 +- **开启**:文件将自动保存至预设的默认存储和目录,不再弹出询问提示。 -开启静默模式后, Bot 会直接保存文件到默认位置, 无需确认. - -在开启静默模式之前, 需要使用 `/storage` 命令设置默认保存位置. +> 开启前请确保已在 `/config` 中设置默认存储。 diff --git a/docs/content/zh/usage/transfer.md b/docs/content/zh/usage/transfer.md index 2041e696..dc7c007a 100644 --- a/docs/content/zh/usage/transfer.md +++ b/docs/content/zh/usage/transfer.md @@ -33,18 +33,17 @@ weight: 8 /transfer local1:/pictures "(?i)\.(jpg|png|gif)$" ``` -Bot 会: +操作流程: 1. 列出源路径下的所有文件 -2. 应用过滤器 (如果提供) -3. 显示文件数量和总大小 -4. 让你选择目标存储 -5. 让你选择目标目录 (如果该存储配置了目录) -6. 开始传输任务 - -注意: - -- 源存储必须支持列举和读取功能 -- 目标存储必须支持写入功能 -- 传输过程显示实时进度 -- 支持取消正在进行的传输任务 +2. 应用过滤器(若提供) +3. 显示文件数量与总大小 +4. 选择目标存储 +5. 选择目标目录(若该存储配置了目录) +6. 开始执行传输 + +说明: + +- 源存储必须支持列举与读取 +- 目标存储必须支持写入 +- 传输过程展示实时进度,支持随时取消 diff --git a/docs/content/zh/usage/watch.md b/docs/content/zh/usage/watch.md index e5bc78ba..4408de69 100644 --- a/docs/content/zh/usage/watch.md +++ b/docs/content/zh/usage/watch.md @@ -6,43 +6,38 @@ weight: 4 # 监听聊天 {{< hint warning >}} -该功能需开启 UserBot 集成. +需要启用 UserBot 集成。 {{< /hint >}} -监听指定聊天的消息, 并自动保存到默认存储中, 遵从存储规则, 并且可以设置过滤器来只保存匹配的消息. +可监听指定聊天中的新消息,支持按过滤器筛选并依照存储规则自动保存。 -监听聊天: +发送 `/watch` 管理监听聊天: -``` -/watch [filter] -``` +- **添加监听** — 显示添加聊天的语法 +- **监听列表** — 列出正在监听的聊天,可逐项停止 +- **通知开关** — 开关所有监听聊天的任务通知 +- **同步对话** — 将 UserBot 的对话列表同步到 peer 缓存 -取消监听: +开始监听: ``` -/unwatch +/watch <聊天ID/用户名> [过滤器] ``` +停止监听:在 `/watch` 的“监听列表”中点击对应聊天即可停止。 + 过滤器类型: ## msgre -正则匹配消息文本, 例如: +按消息文本正则匹配,例如: ``` -/watch 12345678 msgre:.*hello.* +/watch -1002229835658 msgre:.*hello.* ``` -这将会监听 ID 为 12345678 的聊天, 并且只保存消息文本中包含 "hello" 的消息. +这会监听 ID 为 `-1002229835658` 的频道,只保存文本包含 `hello` 的消息。用户私聊使用正数 ID,普通群使用负数 ID,超级群和频道使用 `-100...`。 ## 任务通知 -所有用户的监听任务通知默认关闭. 开启后, 机器人从被监听聊天中保存的每条消息都会在此处收到通知, 并随下载进度更新, 最终显示保存结果: - -``` -/watch notify on -``` - -使用 `/watch notify off` 关闭, 使用 `/watch notify` 查询当前状态. - -通知由机器人发送到你与它的私聊中. 该开关按用户生效, 而非按聊天: `/watch notify off` 会关闭该用户所有被监听聊天的通知. +任务通知默认关闭。在 `/watch` 中切换“通知开关”开启后,Bot 会在私聊中实时推送监听到的消息进度与保存结果。此开关按用户生效,作用于该用户的所有监听聊天。再次点击即可关闭。 diff --git a/docs/content/zh/usage/ytdlp.md b/docs/content/zh/usage/ytdlp.md index cc14abb0..4ac0afb9 100644 --- a/docs/content/zh/usage/ytdlp.md +++ b/docs/content/zh/usage/ytdlp.md @@ -1,51 +1,53 @@ --- -title: "yt-dlp 视频下载" +title: "视频下载 (yt-dlp)" weight: 7 --- -# yt-dlp 视频下载 +# 视频下载 (yt-dlp) {{< hint warning >}} -该功能需要在系统中安装 yt-dlp 命令行工具. +需要系统安装 yt-dlp 命令行工具。 {{< /hint >}} -使用 `/ytdlp` 命令可以下载支持的视频网站的视频和音频, 支持 YouTube、Bilibili、Twitter 等 1000+ 个网站. +视频解析可通过 `/download` 使用。发送支持的视频链接后,选择 **视频(yt-dlp)**。支持 YouTube、Bilibili、Twitter 等主流音视频站点。 ```bash -/ytdlp [url2] [flags...] +/download <链接1> [链接2] [参数...] ``` -示例: +示例: ```bash # 基本下载 -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ # 下载多个视频 -/ytdlp https://www.youtube.com/watch?v=video1 https://www.youtube.com/watch?v=video2 +/download https://www.youtube.com/watch?v=video1 https://www.youtube.com/watch?v=video2 # 使用自定义参数 -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ -f best -/ytdlp https://www.youtube.com/watch?v=dQw4w9WgXcQ --extract-audio --audio-format mp3 +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ -f best +/download https://www.youtube.com/watch?v=dQw4w9WgXcQ --extract-audio --audio-format mp3 ``` -常用参数: +常用参数: -- `-f `: 指定下载格式 (如 `best`, `worst`, `bestvideo+bestaudio`) -- `-o