From af3fa4cfd55b792959b088f6787407637f3ac311 Mon Sep 17 00:00:00 2001 From: sunm2n Date: Wed, 8 Apr 2026 00:16:57 +0900 Subject: [PATCH 001/173] =?UTF-8?q?[Chore]=20=EC=9D=B4=EC=8A=88=20?= =?UTF-8?q?=ED=85=9C=ED=94=8C=EB=A6=BF=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IssueNum #1 --- ...0\353\212\245-\352\260\234\353\260\234.md" | 24 +++++++++++++++++++ ...0\355\203\200-\354\227\205\353\254\264.md" | 23 ++++++++++++++++++ ...54\355\214\251\355\206\240\353\247\201.md" | 24 +++++++++++++++++++ ...4\353\223\234-\354\210\230\354\240\225.md" | 24 +++++++++++++++++++ ...4\352\267\270-\354\210\230\354\240\225.md" | 24 +++++++++++++++++++ ...4\353\223\234-\352\264\200\353\240\250.md" | 24 +++++++++++++++++++ ...4\352\267\270-\354\210\230\354\240\225.md" | 24 +++++++++++++++++++ ...5\355\203\234-\353\263\200\352\262\275.md" | 24 +++++++++++++++++++ ...4\355\212\270-\354\275\224\353\223\234.md" | 24 +++++++++++++++++++ 9 files changed, 215 insertions(+) create mode 100644 ".github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" create mode 100644 ".github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" create mode 100644 ".github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" create mode 100644 ".github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" create mode 100644 ".github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" create mode 100644 ".github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" create mode 100644 ".github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" create mode 100644 ".github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" create mode 100644 ".github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" diff --git "a/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" "b/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" new file mode 100644 index 0000000..8175bae --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\352\270\260\353\212\245-\352\260\234\353\260\234.md" @@ -0,0 +1,24 @@ +--- +name: 기능 개발 +about: 새로운 기능을 추가하거나 수정 +title: "[FEATURE] 기능 개발" +labels: "✨ Feature" +assignees: + +--- + +## 기능 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 구현할 기능에 대한 설계를 마쳤는가? +- [ ] 테스트 코드를 작성했는가? +- [ ] API 명세서(Swagger 등)를 업데이트했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" "b/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" new file mode 100644 index 0000000..17800c0 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\352\270\260\355\203\200-\354\227\205\353\254\264.md" @@ -0,0 +1,23 @@ +--- +name: 기타 업무 +about: 프로덕션 코드가 바뀌지 않고 개발 로직과 상관 없는 가벼운 일들 +title: "[CHORE] 기타 업무" +labels: "🗑 chore" +assignees: + +--- + +## 기타 업무 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 어떤 목적으로 수행되는 작업인지 명확히 작성했는가? +- [ ] 이 작업이 다른 개발 환경에 영향을 주지 않는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" "b/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" new file mode 100644 index 0000000..0e74675 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\246\254\355\214\251\355\206\240\353\247\201.md" @@ -0,0 +1,24 @@ +--- +name: 리팩토링 +about: production 코드를 리팩토링 +title: "[REFACTOR] 리팩토링" +labels: "🔧 Refactor" +assignees: + +--- + +## 리팩토링 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 로직의 가독성이 좋아졌는가? +- [ ] 단일 책임 원칙(SRP)을 잘 따랐는가? +- [ ] 기존 테스트가 모두 통과하는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..77f0fda --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\254\270\354\204\234-\354\236\221\354\204\261-\353\260\217-\354\275\224\353\223\234-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 문서 작성 및 수정 +about: 도큐먼트/문서화 업데이트 +title: "[DOCS] 문서 작성 및 수정" +labels: "📄 Docs" +assignees: + +--- + +## 문서 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 문서의 내용이 최신 코드를 반영하고 있는가? +- [ ] 오타나 잘못된 링크는 없는가? +- [ ] 기술 용어가 일관되게 사용되었는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..b93f346 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\262\204\352\267\270-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 버그 수정 +about: 버그를 고침 +title: "[FIX] 버그 수정" +labels: "🐞 Bugfix" +assignees: + +--- + +## 버그 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 버그가 발생하는 원인을 분석했는가? +- [ ] 버그가 재현되는 테스트 코드를 작성했는가? +- [ ] 수정 사항이 기존 기능에 영향을 주지 않는지 확인했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" "b/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" new file mode 100644 index 0000000..aa696f7 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\353\271\214\353\223\234-\352\264\200\353\240\250.md" @@ -0,0 +1,24 @@ +--- +name: 빌드 관련 +about: 빌드 관련 수정, 모듈 설치 또는 삭제 등 +title: "[BUILD] 빌드 관련" +labels: "🛠 Build" +assignees: + +--- + +## 빌드 관련 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 의존성이 올바르게 추가/삭제 되었는가? +- [ ] 로컬 환경에서 빌드(./gradlew build)가 성공하는가? +- [ ] 환경 변수나 설정 파일(.properties, .yml)의 변경이 필요한가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" "b/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" new file mode 100644 index 0000000..e2a3733 --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\354\213\234\352\270\211\355\225\234-\353\262\204\352\267\270-\354\210\230\354\240\225.md" @@ -0,0 +1,24 @@ +--- +name: 시급한 버그 수정 +about: 시급한 버그를 고침 - 현 production에 critical +title: "[HOTFIX] 버그 수정" +labels: "🐞 HOTFIX" +assignees: + +--- + +## 시급한 버그 수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 버그를 재현할 수 있는 테스트 코드를 작성했는가? +- [ ] 해결책이 기존 코드의 다른 기능에 영향을 주지 않는가? +- [ ] 로그나 에러 메시지를 충분히 확인했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" "b/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" new file mode 100644 index 0000000..ee9a6fb --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\354\275\224\353\223\234-\355\230\225\355\203\234-\353\263\200\352\262\275.md" @@ -0,0 +1,24 @@ +--- +name: 코드 형태 변경 +about: 주석, 프리티어 등 기능의 영향없이 코드 모양새만 변경되는 경우 +title: "[STYLE] 코드 형태 변경" +labels: "💄 Style" +assignees: + +--- + +## 코드 형태 변경 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 비즈니스 로직에 영향이 없는가? +- [ ] 정해진 코드 스타일 가이드를 준수했는가? +- [ ] 불필요한 주석이나 공백을 제거했는가? + +### 참고자료 + + \ No newline at end of file diff --git "a/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" "b/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" new file mode 100644 index 0000000..81a344c --- /dev/null +++ "b/.github/ISSUE_TEMPLATE/\355\205\214\354\212\244\355\212\270-\354\275\224\353\223\234.md" @@ -0,0 +1,24 @@ +--- +name: 테스트 코드 +about: 테스트 코드 추가 및 업데이트 +title: "[TEST] 테스트" +labels: "✅ Test" +assignees: + +--- + +## 테스트 코드 추가/수정 + +### 설명 + + + +### 🗒 체크리스트 + +- [ ] 테스트 케이스가 모든 엣지 케이스를 포함하는가? +- [ ] 테스트가 독립적으로 실행 가능한가? +- [ ] 테스트 코드의 네이밍이 이해하기 쉬운가? + +### 참고자료 + + \ No newline at end of file From 8d9dd0b7bf40b725eca8a96f21cc69c5624341fa Mon Sep 17 00:00:00 2001 From: sunm2n Date: Wed, 8 Apr 2026 00:17:07 +0900 Subject: [PATCH 002/173] =?UTF-8?q?[Chore]=20pr=20=ED=85=9C=ED=94=8C?= =?UTF-8?q?=EB=A6=BF=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IssueNum #1 --- .github/pull_request_template.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..58b9b19 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ +# 요약 + + +# 연관 이슈 및 Close 할 이슈 작성 +(fix #일이삼) + + +-- PR 시 다음과 같이 작성 +#123 + +close #123 + +# Pull Request 체크리스트 + +## TODO + +- [ ] 최종 결과물을 확인했는가? +- [ ] 의미 있는 커밋 메시지를 작성했는가? + \ No newline at end of file From b2d86a75418be6c5fb1850941ebb3c73c18e1b0b Mon Sep 17 00:00:00 2001 From: sunm2n Date: Wed, 8 Apr 2026 00:17:22 +0900 Subject: [PATCH 003/173] =?UTF-8?q?[Chore]=20=EC=9E=90=EB=8F=99=ED=99=94?= =?UTF-8?q?=20=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IssueNum #1 --- .github/workflows/auto-close-issues.yml | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/auto-close-issues.yml diff --git a/.github/workflows/auto-close-issues.yml b/.github/workflows/auto-close-issues.yml new file mode 100644 index 0000000..eeb3692 --- /dev/null +++ b/.github/workflows/auto-close-issues.yml @@ -0,0 +1,57 @@ +name: Auto Close Issues + +on: + pull_request: + types: [closed] + +jobs: + close-issues: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - name: Close linked issues + uses: actions/github-script@v7 + with: + script: | + const prBody = context.payload.pull_request.body || ''; + const prTitle = context.payload.pull_request.title || ''; + + // PR 본문과 제목에서 이슈 번호 찾기 + const issueRegex = /(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)\s+#(\d+)/gi; + const matches = [...prBody.matchAll(issueRegex), ...prTitle.matchAll(issueRegex)]; + + for (const match of matches) { + const issueNumber = parseInt(match[1]); + + try { + // 이슈 상태 확인 + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber + }); + + if (issue.data.state === 'open') { + // 이슈 종료 + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + state: 'closed' + }); + + // 백로그 라벨 추가 (선택사항) + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + labels: ['백로그'] + }); + + console.log(`Closed issue #${issueNumber}`); + } + } catch (error) { + console.log(`Failed to close issue #${issueNumber}: ${error.message}`); + } + } \ No newline at end of file From f687ff1363ee7a72b7f1ad84821912f6e6c9f0c2 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 8 Apr 2026 12:27:20 +0900 Subject: [PATCH 004/173] =?UTF-8?q?[Docs]=20=ED=94=84=EB=A1=9C=EC=A0=9D?= =?UTF-8?q?=ED=8A=B8=20README(=EA=B8=B0=EB=8A=A5=20=EB=AA=85=EC=84=B8)=20?= =?UTF-8?q?=EC=9E=91=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 403 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..547ca8f --- /dev/null +++ b/README.md @@ -0,0 +1,403 @@ +# LinClean-BE-fastapi (URL 보안 엔진) + +**LinClean의 URL 검역 백엔드입니다.**
+카톡·문자·메일로 받은 링크를 열기 전에 4단계 파이프라인으로 검사해 +**안전 / 주의 / 위험** 판정과 그 근거를 산출하는 **상태 비저장(stateless) 분석 전용 서버** 입니다. + +--- + +## 기술 스택 + +| 분류 | 기술 | 비고 | +|------|-------------------------------------------------|------| +| Framework | **FastAPI** | lifespan, async, app factory | +| Language | **Python 3.11+** | 타입 힌트, async/await | +| 로컬 캐시 DB | **SQLite + aiosqlite** | URLhaus 등 외부 피드 캐시 전용 | +| ORM | **SQLAlchemy 2.0 (async)** | DeclarativeBase + naming convention | +| Migration | **Alembic** | SQLite batch mode | +| HTTP Client | **httpx** | 외부 API 호출 (GSB / RDAP / Claude / Spring 콜백) | +| Crawler | **BeautifulSoup4 + requests** | 페이지 본문 추출, 피싱 신호 탐지 | +| Domain Lookup | **RDAP (httpx)** | 도메인 등록일·만료일·레지스트라 조회 | +| Async Queue | **Celery + Redis** | 무거운 분석 작업 큐잉, TTL 캐시 | +| Scheduler | **APScheduler** | URLhaus 주기 동기화, 링크 생존 모니터링 | +| Validation | **Pydantic v2 + pydantic-settings** | 요청·응답·환경변수 | +| Logging | **structlog** | 구조적 로깅 + request_id 자동 바인딩 | +| Lint / Format | **Ruff** | E/F/I/B/SIM/S/UP 룰셋 | +| Type Check | **mypy (strict)** | pydantic 플러그인 | +| Test | **pytest + pytest-asyncio + httpx AsyncClient** | ASGI 테스트 | +| Packaging | **hatchling** | PEP 621 | +| AI | **모델 미정** | 페이지 콘텐츠 정적 분석 보조 | + +--- + +## 디렉토리 구조 + +``` +linclean-fastapi/ +├── app/ +│ ├── main.py # FastAPI 앱 팩토리 + lifespan +│ │ +│ ├── api/ # HTTP 계층 +│ │ ├── deps.py # 공용 의존성 (DBSession 등) +│ │ ├── error_handlers.py # 글로벌 예외 → ErrorResponse 변환 +│ │ └── v1/ +│ │ ├── router.py # v1 라우터 집합 +│ │ └── endpoints/ +│ │ └── health.py # /health, /health/ready +│ │ +│ ├── core/ # 인프라/공통 +│ │ ├── config.py # 환경변수 (pydantic-settings) +│ │ ├── logging.py # structlog + stdlib bridge +│ │ └── exceptions.py # AppError 도메인 예외 계층 +│ │ +│ ├── db/ # 영속성 계층 (외부 피드 캐시 전용) +│ │ ├── base.py # DeclarativeBase + naming convention +│ │ └── session.py # async engine, get_db, SQLite PRAGMA +│ │ +│ ├── middleware/ +│ │ └── request_context.py # X-Request-ID + 구조적 access log +│ │ +│ ├── models/ # SQLAlchemy ORM 모델 (외부 피드 캐시) +│ │ └── __init__.py # Alembic autogen 용 import 모음 +│ │ +│ ├── schemas/ # Pydantic DTO +│ │ └── common.py # HealthResponse, ErrorResponse 등 +│ │ +│ └── services/ # 도메인/비즈니스 로직 +│ (4단계 파이프라인 / 외부 연동 / Spring 콜백이 들어갈 자리) +│ +├── alembic/ # 외부 피드 캐시 스키마 마이그레이션 +│ ├── env.py # SQLite + batch mode 설정 +│ ├── script.py.mako +│ └── versions/ +│ +├── tests/ # pytest 테스트 +│ ├── conftest.py # ASGI AsyncClient + 인메모리 SQLite +│ └── test_health.py +│ +├── docs/ # 기획서 등 설계 문서 +│ └── LinClean_기능_초안.pdf +│ +├── data/ # SQLite 캐시 파일 (gitignore) +│ +├── alembic.ini +├── pyproject.toml # 의존성 + ruff/mypy/pytest 설정 +├── Makefile # install / run / test / migrate ... +├── .pre-commit-config.yaml +├── .env.example +└── README.md +``` + +### 계층별 책임 + +- **`api/`** — HTTP 입출력만 담당. 라우터는 얇게 유지하고, 비즈니스 로직은 + `services/` 에 위임합니다. 의존성(`Depends`)은 `api/deps.py` 에 모아둡니다. +- **`core/`** — 프레임워크에 종속되지 않는 인프라 코드. 설정 로드, 로깅 구성, + 도메인 예외(`AppError`) 등 어디서든 import 해도 안전한 모듈만 둡니다. +- **`db/`** — SQLAlchemy 엔진/세션과 `Base`. **여기서 다루는 것은 외부 위협 + 피드 캐시뿐입니다.** 비즈니스 엔티티(User, Link, Directory 등)는 만들지 + 마세요. SQLite 전용 PRAGMA(`WAL`, `foreign_keys=ON`, `synchronous=NORMAL`) + 가 연결마다 자동 적용됩니다. +- **`models/`** — ORM 모델. 새 모델을 추가하면 반드시 + `app/models/__init__.py` 에서 import 해야 Alembic autogenerate 가 인식합니다. +- **`schemas/`** — 요청·응답 Pydantic 모델. Spring 으로 보내는 분석 결과 DTO + (`AnalysisResultCallback`) 도 여기서 정의합니다. +- **`services/`** — 4단계 파이프라인, 외부 API 클라이언트, 점수 산출, Spring + 콜백 호출, AI 정적 분석 호출 등 모든 비즈니스 로직. `Request` 같은 FastAPI + 객체를 받지 않고 `AsyncSession` / 순수 인자만 받습니다. +- **`middleware/`** — `RequestContextMiddleware` 가 매 요청마다 `X-Request-ID` + 를 생성/전파하고 structlog contextvars 에 바인딩합니다. 응답 헤더로도 echo + 되며 모든 로그 라인에 자동으로 따라붙습니다. +- **`alembic/`** — SQLite 의 제한적 ALTER 지원을 보완하기 위해 + `render_as_batch=True` 로 동작합니다. + +--- + +## 핵심 — URL 안전성 분석 4단계 파이프라인 + +``` +URL 입력 (Spring 으로부터 위임) + │ + ▼ +1단계: URL 정규화 + 단축 URL 언체이닝 ← 스킴·호스트 소문자, 기본 포트/프래그먼트 + │ 제거, 리다이렉트 체인 추적해 최종 URL 확정 + ▼ +2단계: 외부 위협 DB 대조 ← GSB(API) + URLhaus(로컬 SQLite) + │ (최종 URL 기준 대조 — 블랙리스트 매치 시 즉시 short-circuit 가능) + │ + ▼ +3단계: 도메인 휴리스틱 분석 ← RDAP(등록일·레지스트라), 오타 도메인, 패턴 + │ + ▼ +4단계: 페이지 콘텐츠 정적 분석 ← BeautifulSoup + AI API (피싱 신호 추론) + │ + ▼ +종합 위험 점수 산출 → 안전 / 주의 / 위험 판정 + │ + ▼ +Spring `/internal/analysis-result` 콜백 POST +``` + +### 1단계 — URL 정규화 + 단축 URL 언체이닝 + +외부 DB 대조와 도메인 분석이 의미를 가지려면 **어떤 URL 을 검사할지부터 +확정** 해야 합니다. `bit.ly` 같은 단축 URL 상태로 GSB / URLhaus 를 조회하면 +거의 항상 매치되지 않기 때문에, 모든 후속 단계의 입력이 되는 "최종 URL" +을 먼저 만듭니다. + +- **정규화**: 스킴·호스트 소문자화, 기본 포트(`:80` / `:443`) 제거, + 프래그먼트(`#...`) 제거, 퍼센트 인코딩 정돈, 추적 파라미터(`utm_*` 등) + 정책적 제거, IDN(퓨니코드) → 유니코드 정규화 +- **언체이닝**: `HEAD` 요청만으로 리다이렉트 체인(3xx Location) 을 끝까지 + 따라가 **최종 분석 대상 URL** 을 확정합니다. 무한 루프 방지를 위해 방문한 + URL 을 추적하며, hop 수가 5 이상이면 자체적으로 의심 신호로 가산합니다. +- 이후 2~4 단계는 모두 이 **최종 URL** 을 기준으로 동작합니다. + +### 2단계 — 외부 위협 DB 대조 + +1단계에서 확정된 최종 URL 을 두 개의 위협 피드와 병렬로 대조합니다. 자체 +휴리스틱보다 먼저 실행해 이미 알려진 악성 URL 이면 조기에 `danger` 로 +short-circuit 할 수 있습니다. + +| 소스 | 방식 | 응답 시간 | 탐지 대상 | +|------|------|-----------|-----------| +| **Google Safe Browsing** | Google API 실시간 조회 | 100~300ms | 피싱 / 멀웨어 / 소셜 엔지니어링 | +| **URLhaus (abuse.ch)** | CSV → **로컬 SQLite** 캐시 | 1~5ms | 멀웨어 배포 URL | + +URLhaus 데이터는 APScheduler 가 주기적으로 CSV 를 다운로드해 로컬 SQLite 에 +upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조회합니다. 두 소스 +중 하나라도 매치되면 그 자체로 강한 위험 신호이며, 점수 가산과 함께 후속 +단계에 결과를 그대로 전달합니다. + +### 3단계 — 도메인 휴리스틱 분석 + +규칙 기반 점수표로 도메인의 위험 신호를 합산합니다. 도메인 등록 정보는 +**RDAP (RFC 7480~7484)** 로 조회합니다. + +| 검사 항목 | 위험 신호 예시 | 점수 | +|----------|----------------|------| +| 오타 도메인 (레벤슈타인 거리 1~2) | `naverr.com`, `naaver.com` | +30 | +| 신규 도메인 (RDAP 등록 30일 미만) | `created_date` 기준 | +25 | +| IP 직접 접근 | `http://192.168.x.x/login` | +40 | +| 특수문자·하이픈 과다 | `login-secure-naver-auth.com` | +20 | +| 서브도메인 과다 중첩 | `signin.auth.naver.attacker.xyz` | +35 | +| HTTPS 미사용 (로그인 페이지) | `http://` | +15 | +| 합법 호스팅 플랫폼 (가산) | `github.io`, `netlify.app` | +15 | +| 오픈 리다이렉트 파라미터 | `?url=`, `?redirect=` | +35 | + +레벤슈타인 거리 함수는 외부 라이브러리에 의존하지 않고 직접 구현합니다 (DP). +RDAP 응답은 도메인 단위로 인메모리 캐싱(24h) 하여 동일 도메인 재조회 비용을 +줄입니다. + +### 4단계 — 페이지 콘텐츠 정적 분석 + +`requests + BeautifulSoup` 으로 실제 페이지를 크롤링해 HTML 구조를 추출하고, +**AI(모델 미정) API** 가 피싱 신호를 정적으로 추론합니다. AI 는 본 엔진 안에서 +이 정적 분석 단계에서만 사용됩니다. + +추출 / 점수화하는 신호: + +- **로그인 폼 + 브랜드 위장**: `` 가 있고 title 에는 + 유명 브랜드명이 있는데 도메인은 그 브랜드와 무관 (+50) +- **브랜드 로고 이미지 위장**: `...` 의 alt 텍스트와 도메인 불일치 (+30) +- **`meta refresh` 자동 리다이렉트** (+20) +- **외부 링크 비율 과다**: 80% 이상이 외부 도메인이면 (+15) +- **AI 정적 추론**: 추출된 텍스트·폼·메타데이터를 AI 에게 넘겨 "이 페이지가 + 특정 브랜드를 사칭하거나 자격 증명을 탈취하려 하는지" 여부와 근거 텍스트를 + 반환받아 점수에 반영 +- 페이지 접근 실패 시 +20 점 (의심 신호로 취급) + +### 최종 점수 산출 + +각 단계의 점수를 합산해 다음 구간으로 매핑합니다. + +| 합산 점수 | verdict | UI | +|-----------|---------|----| +| 0 ~ 30 | **safe** (안전) | 초록 — 정상적으로 열기 가능 | +| 31 ~ 60 | **caution** (주의) | 노랑 — 이유 표시 후 사용자 판단 | +| 61 이상 | **danger** (위험) | 빨강 — "피싱 의심, 열지 마세요" | + +판정 결과와 함께 **"왜 위험한지 이유"** (`reasons` 배열) 를 반환해 사용자가 +판단 근거를 이해할 수 있게 합니다. + +--- + + +## 시스템 한계와 보완 방안 + +어떤 정적 분석 시스템도 100% 탐지는 불가능합니다. 본 엔진이 인지하고 있는 +주요 한계와 대응 방안은 다음과 같습니다. + +| 공격 방식 | 한계 | 대응 방안 | +|----------|------|----------| +| 지연 활성화 (TOCTOU) | 분석 후 피싱 페이지로 교체 | 클릭 직전 재검사 + 리마인드 시 재검사 | +| 봇 탐지 | 크롤러엔 정상 페이지 반환 | User-Agent 주기적 교체 | +| 합법 도메인 악용 (`github.io` 등) | 신뢰 도메인 위에 피싱 호스팅 | 호스팅 플랫폼 가산 점수 | +| 리버스 프록시 | 진짜 사이트를 실시간 프록시 | 도메인 불일치 경고 | +| 오픈 리다이렉트 | 합법 도메인 경유 후 피싱 | 리다이렉트 파라미터 감지 | + +--- + +## 두 백엔드 간 통신 규약 + +분석 엔진은 **요청-응답 동기 호출이 아니라 콜백** 방식으로 Spring 과 통신합니다. +Spring 이 분석 위임 요청을 보내면, 본 엔진은 즉시 `analysisId` 를 반환하고 +실제 결과는 분석이 끝난 뒤 Spring 의 내부 콜백 엔드포인트로 POST 합니다. + +``` +Spring FastAPI (본 엔진) + │ │ + │ POST /api/v1/analyze │ + │ { analysisId, url } │ + ├──────────────────────────────────────►│ + │ │ (4단계 파이프라인 비동기 실행) + │◄──── 202 Accepted │ + │ { analysisId, status: "queued" } │ + │ │ + │ │ (분석 완료) + │ POST /internal/analysis-result │ + │ X-Internal-Token: │ + │ { ... AnalysisResultCallback ... } │ + │◄──────────────────────────────────────┤ + │ │ + │ 200 OK { received: true } │ + ├──────────────────────────────────────►│ +``` + +모든 호출에는 `X-Internal-Token` 헤더가 포함되어야 하며, Spring 은 이 헤더가 +없거나 값이 일치하지 않는 요청을 거부합니다. 외부에서 `/internal/*` 경로를 +직접 호출할 수 없게 하기 위함입니다. + +### Spring 콜백 메시지 정의 + +**Endpoint:** `POST {SPRING_INTERNAL_URL}/internal/analysis-result` + +**Headers:** + +| 헤더 | 필수 | 설명 | +|------|------|------| +| `Content-Type` | ✓ | `application/json` | +| `X-Internal-Token` | ✓ | 두 백엔드가 공유하는 사전 합의 토큰 | +| `X-Request-ID` | ✓ | 원 요청의 request id 를 그대로 echo (분산 추적용) | + +**Body — 성공 (`AnalysisResultCallback`):** + +```json +{ + "analysisId": "9f0e0e3a-2c1f-4b76-9d3e-0e0a5a1cf2a1", + "requestId": "b4c3a9e7-7c2a-4a1d-9f0b-1c2d3e4f5a6b", + "status": "succeeded", + "originalUrl": "https://bit.ly/abc123", + "finalUrl": "https://login-secure-naver-auth.com/signin", + "verdict": "danger", + "score": 82, + "reasons": [ + { + "code": "GSB_MATCH", + "stage": 2, + "weight": 50, + "message": "Google Safe Browsing 에 SOCIAL_ENGINEERING 으로 등재된 URL" + }, + { + "code": "TYPO_DOMAIN", + "stage": 3, + "weight": 30, + "message": "naver.com 과 레벤슈타인 거리 2 의 유사 도메인" + }, + { + "code": "BRAND_IMPERSONATION_FORM", + "stage": 4, + "weight": 50, + "message": "비공식 도메인에서 NAVER 로그인 폼을 노출" + } + ], + "stages": { + "externalDb": { + "gsb": { "isThreat": true, "matchedTypes": ["SOCIAL_ENGINEERING"] }, + "urlhaus": { "isThreat": false, "host": "bit.ly" } + }, + "unchain": { + "hops": 2, + "chain": [ + "https://bit.ly/abc123", + "https://login-secure-naver-auth.com/signin" + ] + }, + "domainHeuristic": { + "rdap": { + "domain": "login-secure-naver-auth.com", + "registrar": "NameCheap, Inc.", + "createdDate": "2026-03-22T00:00:00Z", + "domainAgeDays": 16, + "isNewDomain": true + }, + "signals": ["TYPO_DOMAIN", "NEW_DOMAIN", "HYPHEN_OVERUSE"] + }, + "contentAnalysis": { + "fetched": true, + "hasPasswordField": true, + "aiVerdict": "phishing", + "aiReason": "페이지 제목과 로고 alt 텍스트가 NAVER 를 사칭하나, 도메인 소유자 정보가 일치하지 않음" + } + }, + "summary": "NAVER 로그인 페이지를 사칭하는 신규 도메인입니다. 절대 자격 증명을 입력하지 마세요.", + "engineVersion": "0.1.0", + "analyzedAt": "2026-04-07T05:42:11Z", + "elapsedMs": 1843 +} +``` + +**Body — 실패:** + +```json +{ + "analysisId": "9f0e0e3a-2c1f-4b76-9d3e-0e0a5a1cf2a1", + "requestId": "b4c3a9e7-7c2a-4a1d-9f0b-1c2d3e4f5a6b", + "status": "failed", + "originalUrl": "https://bit.ly/abc123", + "error": { + "code": "PIPELINE_TIMEOUT", + "stage": 4, + "message": "콘텐츠 분석 단계에서 30s 타임아웃 초과" + }, + "engineVersion": "0.1.0", + "analyzedAt": "2026-04-07T05:42:41Z", + "elapsedMs": 30000 +} +``` + +**필드 정의:** + +| 필드 | 타입 | 필수 | 설명 | +|------|------|------|------| +| `analysisId` | string (uuid) | ✓ | Spring 이 분석 위임 시 발급한 식별자. 콜백에서 그대로 echo | +| `requestId` | string (uuid) | ✓ | 분산 추적용 request id (`X-Request-ID` 헤더와 동일) | +| `status` | enum | ✓ | `succeeded` / `failed` | +| `originalUrl` | string | ✓ | Spring 에게서 받은 원본 URL | +| `finalUrl` | string | succeeded | 1·2단계를 거친 뒤 확정된 최종 분석 대상 URL | +| `verdict` | enum | succeeded | `safe` / `caution` / `danger` | +| `score` | int (0~100) | succeeded | 4단계 합산 점수 (100 cap) | +| `reasons[]` | array | succeeded | 점수에 기여한 신호들. UI 의 "이유" 배지에 그대로 사용 가능 | +| `reasons[].code` | string | ✓ | 머신 식별 코드 (예: `GSB_MATCH`, `TYPO_DOMAIN`, `NEW_DOMAIN`) | +| `reasons[].stage` | int (1~4) | ✓ | 어느 파이프라인 단계에서 발생했는지 | +| `reasons[].weight` | int | ✓ | 해당 신호가 더한 점수 | +| `reasons[].message` | string | ✓ | 사용자에게 보여줄 한국어 설명 | +| `stages` | object | succeeded | 단계별 원시 결과 (감사·디버깅 용) | +| `summary` | string | succeeded | Claude 가 만든 1~3문장 요약. UI 의 "이 링크는…" 카피로 사용 | +| `engineVersion` | string | ✓ | 본 엔진 버전. Spring 이 결과의 호환성을 판단할 때 사용 | +| `analyzedAt` | string (ISO8601 UTC) | ✓ | 분석 종료 시각 | +| `elapsedMs` | int | ✓ | 분석에 소요된 wall-clock 시간 (밀리초) | +| `error` | object | failed | 실패 원인 | + +**Spring 측 응답:** + +```json +HTTP/1.1 200 OK +{ "received": true } +``` + +- `200 OK` 외 응답 또는 네트워크 오류 시 본 엔진은 지수 백오프로 최대 3회 + 재시도합니다. 그 후에도 실패하면 dead-letter 로그에 적재하고 운영자가 + 수동 재처리할 수 있게 합니다. +- 동일 `analysisId` 의 콜백이 중복 도착할 수 있으므로, **Spring 측 처리는 + 멱등(idempotent) 해야 합니다.** \ No newline at end of file From ecf8f7df64bb458a9a110f3638b6009205b1e0ce Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 8 Apr 2026 14:33:04 +0900 Subject: [PATCH 005/173] =?UTF-8?q?[Docs]=20README=EC=97=90=EC=84=9C=20X-I?= =?UTF-8?q?nternal-Token=20->=20X-Internal-Api-Key=EB=A1=9C=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 547ca8f..0cf0009 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,7 @@ Spring FastAPI (본 엔진) │ │ │ │ (분석 완료) │ POST /internal/analysis-result │ - │ X-Internal-Token: │ + │ X-Internal-Api-Key: │ │ { ... AnalysisResultCallback ... } │ │◄──────────────────────────────────────┤ │ │ @@ -264,9 +264,11 @@ Spring FastAPI (본 엔진) ├──────────────────────────────────────►│ ``` -모든 호출에는 `X-Internal-Token` 헤더가 포함되어야 하며, Spring 은 이 헤더가 +모든 호출에는 `X-Internal-Api-Key` 헤더가 포함되어야 하며, Spring 은 이 헤더가 없거나 값이 일치하지 않는 요청을 거부합니다. 외부에서 `/internal/*` 경로를 -직접 호출할 수 없게 하기 위함입니다. +직접 호출할 수 없게 하기 위함입니다. 이 값은 동적으로 발급·만료되는 토큰이 +아니라, 두 백엔드가 환경변수로 공유하는 **정적 사전 공유 키(pre-shared key)** +입니다. ### Spring 콜백 메시지 정의 @@ -277,7 +279,7 @@ Spring FastAPI (본 엔진) | 헤더 | 필수 | 설명 | |------|------|------| | `Content-Type` | ✓ | `application/json` | -| `X-Internal-Token` | ✓ | 두 백엔드가 공유하는 사전 합의 토큰 | +| `X-Internal-Api-Key` | ✓ | 두 백엔드가 환경변수로 공유하는 사전 공유 키 | | `X-Request-ID` | ✓ | 원 요청의 request id 를 그대로 echo (분산 추적용) | **Body — 성공 (`AnalysisResultCallback`):** From 786174c784a2d845453b38d3f73a5b0faa1ecfcb Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 19:38:13 +0900 Subject: [PATCH 006/173] =?UTF-8?q?[Feat]=20URL=20=EC=A0=95=EA=B7=9C?= =?UTF-8?q?=ED=99=94=20=EC=84=A4=EC=A0=95/=EC=98=88=EC=99=B8/=EC=8A=A4?= =?UTF-8?q?=ED=82=A4=EB=A7=88=20=EC=A0=95=EC=9D=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 3 +++ app/core/exceptions.py | 6 ++++++ app/schemas/analysis.py | 8 ++++++++ 3 files changed, 17 insertions(+) create mode 100644 app/schemas/analysis.py diff --git a/app/core/config.py b/app/core/config.py index ed5ae8c..aa58a9d 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -75,6 +75,9 @@ def alembic_database_url(self) -> str: rdap_cache_ttl_seconds: int = 60 * 60 * 24 # 24h rdap_new_domain_threshold_days: int = 30 + # ---- Normalizer -------------------------------------------------------- + normalizer_max_url_length: int = 2048 + # ---- Scoring ----------------------------------------------------------- score_weight_gsb: int = 50 score_weight_urlhaus: int = 50 diff --git a/app/core/exceptions.py b/app/core/exceptions.py index b52fea4..7bdf0b9 100644 --- a/app/core/exceptions.py +++ b/app/core/exceptions.py @@ -53,3 +53,9 @@ class ValidationError(AppError): status_code = 422 code = "validation_error" message = "Invalid input." + + +class NormalizationError(AppError): + status_code = 422 + code = "normalization_error" + message = "URL normalization failed." diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py new file mode 100644 index 0000000..951bd3c --- /dev/null +++ b/app/schemas/analysis.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel, Field + + +class NormalizeResult(BaseModel): + """1단계 — URL 정규화(Canonicalization) 결과.""" + + original_url: str = Field(description="입력 원본 URL (trim 후)") + normalized_url: str = Field(description="정규화된 URL") From b7ebd8d9b98aff219f875045eacd7ebb94f58723 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 19:39:25 +0900 Subject: [PATCH 007/173] =?UTF-8?q?[Feat]=20URL=20=EC=A0=95=EA=B7=9C?= =?UTF-8?q?=ED=99=94=20service=20=EC=BD=94=EB=93=9C=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/__init__.py | 3 + app/services/normalizer/normalize.py | 194 +++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 app/services/normalizer/__init__.py create mode 100644 app/services/normalizer/normalize.py diff --git a/app/services/normalizer/__init__.py b/app/services/normalizer/__init__.py new file mode 100644 index 0000000..d975755 --- /dev/null +++ b/app/services/normalizer/__init__.py @@ -0,0 +1,3 @@ +from app.services.normalizer.normalize import normalize_url + +__all__ = ["normalize_url"] diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py new file mode 100644 index 0000000..d82bc0c --- /dev/null +++ b/app/services/normalizer/normalize.py @@ -0,0 +1,194 @@ +"""URL 정규화(Canonicalization) 모듈. + +파이프라인 1단계 — 후속 단계(위협 DB 대조·휴리스틱·콘텐츠 분석)가 +동일한 기준으로 URL을 비교할 수 있도록 canonical form으로 변환합니다. + +수행 항목: + - 입력 검증 (빈 문자열, 공백 trim, 제어 문자 제거, 최대 길이 제한) + - 스킴 검증 / 기본값 보정 (http, https, ftp만 허용) + - 스킴·호스트 소문자화 (경로는 대소문자 보존) + - userinfo 제거 (피싱 방지 — google.com@evil.com 같은 트릭 차단) + - 기본 포트 제거 (:80, :443, :21) + - 프래그먼트 제거 + - 퍼센트 인코딩 정돈 (unreserved 디코딩, reserved 대문자 hex 통일) + - 경로 정규화 (빈 경로 → "/", dot-segment 해소, 연속 슬래시 축소) + - IDN 정규화 (유니코드 → 퓨니코드 통일, 외부 위협 DB 호환) + - 원본 보존 (NormalizeResult.original_url) +""" + +from __future__ import annotations + +import re +import unicodedata +from urllib.parse import urlparse, urlunparse + +from app.core.config import settings +from app.core.exceptions import NormalizationError +from app.schemas.analysis import NormalizeResult + +_DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443, "ftp": 21} +_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https", "ftp"}) + +# RFC 3986 §2.3 — unreserved characters +_UNRESERVED: frozenset[str] = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" +) + +# 제어 문자 패턴 (C0, C1, DEL + 유니코드 제어 카테고리) +_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") + +# percent-encoded triplet 패턴 +_PCT_ENCODED_RE = re.compile(r"%([0-9A-Fa-f]{2})") + + +def normalize_url(raw_url: str) -> NormalizeResult: + """원본 URL을 canonical form으로 정규화합니다. + + Returns: + NormalizeResult: original_url(trim 후 원본)과 normalized_url. + + Raises: + NormalizationError: 검증 실패 시. + """ + # ---- 입력 검증 ---------------------------------------------------------- + original = raw_url.strip() + if not original: + raise NormalizationError(message="빈 URL입니다.") + + # 제어 문자 제거 + cleaned = _CONTROL_CHAR_RE.sub("", original) + # NFC 정규화 (유니코드 합성) + cleaned = unicodedata.normalize("NFC", cleaned) + + # ---- 스킴 보정 (길이 검사보다 먼저 수행) --------------------------------- + if "://" not in cleaned: + cleaned = "https://" + cleaned + + # ---- 최대 길이 검사 (스킴 보정 후) --------------------------------------- + max_len = settings.normalizer_max_url_length + if len(cleaned) > max_len: + raise NormalizationError( + message=f"URL이 최대 길이({max_len}자)를 초과합니다.", + ) + + try: + parsed = urlparse(cleaned) + except ValueError as e: + raise NormalizationError(message=f"URL 파싱 실패: {e}") from e + + scheme = parsed.scheme.lower() + if scheme not in _ALLOWED_SCHEMES: + raise NormalizationError(message=f"지원하지 않는 스킴: {scheme}") + + # ---- 호스트 소문자화 + IDN 정규화 ---------------------------------------- + hostname = parsed.hostname or "" + hostname = _normalize_idn(hostname.lower()) + if not hostname: + raise NormalizationError(message="호스트가 비어있습니다.") + + # ---- userinfo 제거 (보안) ----------------------------------------------- + # RFC 3986 §3.2.1의 userinfo (user:pass@)를 의도적으로 삭제합니다. + # 사유: "https://google.com@evil.com/" 처럼 합법 도메인으로 위장하는 + # 피싱 트릭에 사용되며, 브라우저도 userinfo를 무시하거나 경고합니다. + # parsed.hostname이 실제 호스트(evil.com)를 정확히 반환하므로 안전합니다. + + # ---- 기본 포트 제거 ------------------------------------------------------ + port = parsed.port + if port and port == _DEFAULT_PORTS.get(scheme): + port = None + netloc = hostname if port is None else f"{hostname}:{port}" + + # ---- 경로 정규화 (대소문자 보존) ----------------------------------------- + path = _normalize_path(parsed.path) + + # ---- params: 퍼센트 인코딩 정돈 ----------------------------------------- + params = _normalize_pct_encoding(parsed.params) + + # ---- 쿼리: 퍼센트 인코딩 정돈 ------------------------------------------- + query = _normalize_pct_encoding(parsed.query) + + # ---- 프래그먼트 제거 ----------------------------------------------------- + normalized = urlunparse((scheme, netloc, path, params, query, "")) + + return NormalizeResult(original_url=original, normalized_url=normalized) + + +# --------------------------------------------------------------------------- +# 내부 헬퍼 +# --------------------------------------------------------------------------- + + +def _normalize_idn(hostname: str) -> str: + """호스트를 퓨니코드(ASCII-compatible encoding)로 통일합니다. + + GSB·URLhaus 등 외부 위협 DB가 퓨니코드 형태로 URL을 저장하므로, + 유니코드 도메인을 퓨니코드로 변환해야 매칭이 보장됩니다. + 이미 퓨니코드(ASCII)이면 그대로 반환합니다. + """ + try: + return hostname.encode("idna").decode("ascii") + except (UnicodeError, UnicodeDecodeError): + return hostname + + +def _normalize_path(raw_path: str) -> str: + """경로를 정규화합니다. + + - 빈 경로 → "/" + - 퍼센트 인코딩 정돈 + - dot-segment 해소 (RFC 3986 §5.2.4) + - 연속 슬래시 축소 + """ + path = raw_path or "/" + + # 퍼센트 인코딩 정돈 (경로는 대소문자 보존하므로 safe에 슬래시 포함) + path = _normalize_pct_encoding(path) + + # 연속 슬래시 축소 — 보안 목적의 의도적 정규화. + # RFC 상 //a와 /a는 다른 리소스이지만, 공격자가 //를 삽입해 + # 위협 DB 패턴 매칭을 우회하는 것을 방지합니다. + path = re.sub(r"/{2,}", "/", path) + + # dot-segment 해소 + path = _resolve_dot_segments(path) + + return path or "/" + + +def _resolve_dot_segments(path: str) -> str: + """RFC 3986 §5.2.4 — dot-segment를 해소합니다.""" + segments = path.split("/") + resolved: list[str] = [] + for seg in segments: + if seg == ".": + continue + if seg == "..": + if resolved and resolved[-1] != "": + resolved.pop() + else: + resolved.append(seg) + # 원래 절대 경로면 선행 빈 문자열 보장 + if path.startswith("/") and (not resolved or resolved[0] != ""): + resolved.insert(0, "") + return "/".join(resolved) + + +def _normalize_pct_encoding(value: str) -> str: + """퍼센트 인코딩을 정돈합니다. + + - unreserved 문자(%41 → A): 디코딩 + - reserved / 기타 문자: hex를 대문자로 통일 (%2f → %2F) + """ + if not value: + return value + + def _replace(match: re.Match[str]) -> str: + hex_str = match.group(1) + char = chr(int(hex_str, 16)) + # unreserved면 디코딩 + if char in _UNRESERVED: + return char + # 그 외는 대문자 hex 통일 + return f"%{hex_str.upper()}" + + return _PCT_ENCODED_RE.sub(_replace, value) From e62d86e698a76d8e21eaf92670b36fb052607641 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 19:40:11 +0900 Subject: [PATCH 008/173] =?UTF-8?q?[Test]=20URL=20=EC=A0=95=EA=B7=9C?= =?UTF-8?q?=ED=99=94=20=EB=8B=A8=EC=9C=84=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20?= =?UTF-8?q?=EB=B0=8F=20=EB=8D=B0=EB=AA=A8=20=EC=8A=A4=ED=81=AC=EB=A6=BD?= =?UTF-8?q?=ED=8A=B8=20=EC=9E=91=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/__init__.py | 0 tests/conftest.py | 1 + tests/demo_normalize.py | 86 +++++++ tests/services/__init__.py | 0 tests/services/normalizer/__init__.py | 0 tests/services/normalizer/test_normalize.py | 253 ++++++++++++++++++++ 6 files changed, 340 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/demo_normalize.py create mode 100644 tests/services/__init__.py create mode 100644 tests/services/normalizer/__init__.py create mode 100644 tests/services/normalizer/test_normalize.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..dc723da --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +"""공용 테스트 픽스처.""" diff --git a/tests/demo_normalize.py b/tests/demo_normalize.py new file mode 100644 index 0000000..1c66dbf --- /dev/null +++ b/tests/demo_normalize.py @@ -0,0 +1,86 @@ +"""URL 정규화 데모 + +Usage: + python -m tests.demo_normalize +""" + +from __future__ import annotations + +from app.core.exceptions import NormalizationError +from app.services.normalizer import normalize_url + +DIVIDER = "─" * 72 + +CASES: list[tuple[str, str]] = [ + # (설명, 입력 URL) + ("스킴·호스트 소문자화 + 경로 대소문자 보존", "HTTP://EXAMPLE.COM/Path/To/Page"), + ("기본 포트 제거 (80, 443)", "https://example.com:443/secure"), + ("비기본 포트 유지", "https://example.com:8080/api"), + ("프래그먼트 제거", "https://example.com/page#section-2"), + ("스킴 없으면 https 보정", "example.com/hello"), + ("퍼센트 인코딩: unreserved 디코딩 (%7E→~)", "https://example.com/%7Euser/%41%42"), + ("퍼센트 인코딩: reserved 대문자 hex 통일 (%2f→%2F)", "https://example.com/a%2fb%3fc"), + ("경로 dot-segment 해소 (/../)", "https://example.com/a/b/../c/./d"), + ("연속 슬래시 축소 (보안: DB 매칭 우회 방지)", "https://example.com//a///b"), + ("userinfo 제거 (피싱: google.com@evil.com)", "https://google.com@evil.com/login"), + ("params 퍼센트 인코딩 정돈", "https://example.com/path;%7eparam?q=1"), + ("IDN 유니코드 → 퓨니코드 통일 (외부 DB 호환)", "https://☃.com/path"), + ("IDN 퓨니코드 유지", "https://xn--n3h.com/path"), + ("제어 문자 제거", "https://exam\x00ple.com/pa\x01th"), + ("공백 trim", " https://example.com/page "), + ("종합", " HTTP://EXAMPLE.COM:80/a/b/../c/./d?q=%7e&x=%2f#top "), +] + +ERROR_CASES: list[tuple[str, str]] = [ + ("빈 문자열", ""), + ("공백만", " "), + ("지원하지 않는 스킴", "ftp2://example.com"), + ("호스트 없음", "https://"), + ("최대 길이 초과", "https://example.com/" + "a" * 2048), +] + + +def main() -> None: + print() + print("╔══════════════════════════════════════════════════════════════════════╗") + print("║ LinClean — URL Normalization Test ║") + print("╚══════════════════════════════════════════════════════════════════════╝") + + # ── 정상 케이스 ────────────────────────────────────────────────────────── + print() + print("■ 정상 케이스") + print(DIVIDER) + + for i, (desc, raw) in enumerate(CASES, 1): + result = normalize_url(raw) + # 제어 문자가 포함된 입력은 repr로 표시 + display_input = repr(raw) if any(ord(c) < 32 for c in raw) else raw + print(f" [{i:02d}] {desc}") + print(f" 입력 → {display_input}") + print(f" 원본 → {result.original_url}") + print(f" 정규화 → {result.normalized_url}") + print(DIVIDER) + + # ── 에러 케이스 ────────────────────────────────────────────────────────── + print() + print("■ 에러 케이스 (NormalizationError)") + print(DIVIDER) + + for i, (desc, raw) in enumerate(ERROR_CASES, 1): + display_input = repr(raw) if len(raw) > 60 else (repr(raw) if not raw.strip() else raw) + try: + normalize_url(raw) + print(f" [{i:02d}] {desc}") + print(f" 입력 → {display_input}") + print(f" 결과 → ⚠ 예외가 발생하지 않음!") + except NormalizationError as e: + print(f" [{i:02d}] {desc}") + print(f" 입력 → {display_input}") + print(f" 에러 → {e.message}") + print(DIVIDER) + + print() + + +if __name__ == "__main__": + main() diff --git a/tests/services/__init__.py b/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/normalizer/__init__.py b/tests/services/normalizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py new file mode 100644 index 0000000..ad18e2c --- /dev/null +++ b/tests/services/normalizer/test_normalize.py @@ -0,0 +1,253 @@ +"""normalize_url 단위 테스트.""" + +import pytest +from app.core.exceptions import NormalizationError +from app.services.normalizer.normalize import normalize_url + +# ---- 입력 검증 --------------------------------------------------------------- + + +class TestInputValidation: + def test_empty_string_raises(self) -> None: + with pytest.raises(NormalizationError, match="빈 URL"): + normalize_url("") + + def test_whitespace_only_raises(self) -> None: + with pytest.raises(NormalizationError, match="빈 URL"): + normalize_url(" ") + + def test_whitespace_trimmed(self) -> None: + result = normalize_url(" https://example.com/ ") + assert result.original_url == "https://example.com/" + assert result.normalized_url == "https://example.com/" + + def test_control_characters_removed(self) -> None: + result = normalize_url("https://example\x00.com/pa\x01th") + assert "\x00" not in result.normalized_url + assert "\x01" not in result.normalized_url + assert "example.com" in result.normalized_url + + def test_max_length_exceeded_raises(self) -> None: + long_url = "https://example.com/" + "a" * 2048 + with pytest.raises(NormalizationError, match="최대 길이"): + normalize_url(long_url) + + def test_max_length_boundary_passes(self) -> None: + # 정확히 2048자 — 통과해야 함 + padding = 2048 - len("https://example.com/") + url = "https://example.com/" + "a" * padding + result = normalize_url(url) + assert result.normalized_url.startswith("https://example.com/") + + def test_max_length_checked_after_scheme_prepend(self) -> None: + # 스킴 없이 입력하면 "https://" 가 붙으므로, 보정 후 길이로 검사해야 함 + padding = 2048 - len("https://") - len("example.com/") + url_without_scheme = "example.com/" + "a" * padding + # 스킴 보정 후 정확히 2048자 → 통과 + result = normalize_url(url_without_scheme) + assert result.normalized_url.startswith("https://example.com/") + + +# ---- 원본 보존 --------------------------------------------------------------- + + +class TestOriginalPreservation: + def test_original_url_preserved_after_trim(self) -> None: + result = normalize_url(" HTTP://EXAMPLE.COM/Path ") + assert result.original_url == "HTTP://EXAMPLE.COM/Path" + + def test_original_differs_from_normalized(self) -> None: + result = normalize_url("HTTP://EXAMPLE.COM:80/Path#frag") + assert result.original_url == "HTTP://EXAMPLE.COM:80/Path#frag" + assert result.normalized_url == "http://example.com/Path" + + +# ---- 스킴 ------------------------------------------------------------------- + + +class TestScheme: + def test_lowercase_scheme(self) -> None: + result = normalize_url("HTTP://Example.com") + assert result.normalized_url == "http://example.com/" + + def test_https_preserved(self) -> None: + result = normalize_url("HTTPS://Example.com/path") + assert result.normalized_url == "https://example.com/path" + + def test_no_scheme_defaults_to_https(self) -> None: + result = normalize_url("example.com/path") + assert result.normalized_url == "https://example.com/path" + + def test_unsupported_scheme_raises(self) -> None: + with pytest.raises(NormalizationError, match="지원하지 않는 스킴"): + normalize_url("ftp2://example.com") + + def test_ftp_allowed(self) -> None: + result = normalize_url("FTP://files.example.com/pub") + assert result.normalized_url.startswith("ftp://") + + +# ---- 호스트 ----------------------------------------------------------------- + + +class TestHost: + def test_lowercase_host(self) -> None: + result = normalize_url("https://EXAMPLE.COM/") + assert result.normalized_url == "https://example.com/" + + def test_scheme_only_raises(self) -> None: + with pytest.raises(NormalizationError, match="호스트가 비어있습니다"): + normalize_url("https://") + + def test_userinfo_stripped_for_phishing(self) -> None: + # google.com@evil.com → 실제 호스트는 evil.com + result = normalize_url("https://google.com@evil.com/path") + assert "evil.com" in result.normalized_url + assert "google.com@" not in result.normalized_url + + def test_basic_auth_userinfo_stripped(self) -> None: + result = normalize_url("https://user:pass@example.com/api") + assert "user:pass@" not in result.normalized_url + assert "example.com/api" in result.normalized_url + + +# ---- 포트 ------------------------------------------------------------------- + + +class TestPort: + def test_default_http_port_removed(self) -> None: + result = normalize_url("http://example.com:80/path") + assert result.normalized_url == "http://example.com/path" + + def test_default_https_port_removed(self) -> None: + result = normalize_url("https://example.com:443/path") + assert result.normalized_url == "https://example.com/path" + + def test_default_ftp_port_removed(self) -> None: + result = normalize_url("ftp://example.com:21/file") + assert result.normalized_url == "ftp://example.com/file" + + def test_non_default_port_kept(self) -> None: + result = normalize_url("https://example.com:8080/") + assert ":8080" in result.normalized_url + + +# ---- 경로 ------------------------------------------------------------------- + + +class TestPath: + def test_empty_path_becomes_slash(self) -> None: + result = normalize_url("https://example.com") + assert result.normalized_url == "https://example.com/" + + def test_path_case_preserved(self) -> None: + result = normalize_url("https://example.com/Path/To/Page") + assert "/Path/To/Page" in result.normalized_url + + def test_double_slash_collapsed(self) -> None: + # 보안: 공격자가 //를 삽입해 위협 DB 패턴 매칭을 우회하는 것을 방지 + result = normalize_url("https://example.com//a//b") + assert "//" not in result.normalized_url.split("://", 1)[1] + + def test_dot_segment_resolved(self) -> None: + result = normalize_url("https://example.com/a/b/../c") + assert result.normalized_url == "https://example.com/a/c" + + def test_single_dot_resolved(self) -> None: + result = normalize_url("https://example.com/a/./b") + assert result.normalized_url == "https://example.com/a/b" + + def test_dot_beyond_root(self) -> None: + result = normalize_url("https://example.com/../a") + assert result.normalized_url == "https://example.com/a" + + def test_complex_dot_segments(self) -> None: + result = normalize_url("https://example.com/a/b/c/../../d") + assert result.normalized_url == "https://example.com/a/d" + + +# ---- 프래그먼트 ------------------------------------------------------------- + + +class TestFragment: + def test_fragment_removed(self) -> None: + result = normalize_url("https://example.com/page#section") + assert "#" not in result.normalized_url + + def test_fragment_with_query(self) -> None: + result = normalize_url("https://example.com/page?q=1#top") + assert "#" not in result.normalized_url + assert "q=1" in result.normalized_url + + +# ---- 퍼센트 인코딩 ---------------------------------------------------------- + + +class TestPercentEncoding: + def test_unreserved_decoded(self) -> None: + # %41 = 'A' (unreserved) → 디코딩 + result = normalize_url("https://example.com/%41%42%43") + assert "/ABC" in result.normalized_url + + def test_tilde_decoded(self) -> None: + # %7E = '~' (unreserved) + result = normalize_url("https://example.com/%7Euser") + assert "/~user" in result.normalized_url + + def test_reserved_hex_uppercased(self) -> None: + # %2f = '/' (reserved) → %2F 대문자 통일 + result = normalize_url("https://example.com/path%2fmore") + assert "%2F" in result.normalized_url + + def test_already_uppercase_hex_unchanged(self) -> None: + result = normalize_url("https://example.com/path%2Fmore") + assert "%2F" in result.normalized_url + + def test_space_encoding_uppercased(self) -> None: + # %20 = ' ' (not unreserved) → %20 유지 (이미 대문자) + result = normalize_url("https://example.com/my%20page") + assert "%20" in result.normalized_url + + def test_query_encoding_normalized(self) -> None: + result = normalize_url("https://example.com/?q=%61%62%63") + assert "q=abc" in result.normalized_url + + def test_params_encoding_normalized(self) -> None: + # path params(;)도 path, query와 동일하게 퍼센트 인코딩 정돈 + result = normalize_url("https://example.com/path;%7eparam?q=1") + assert ";~param" in result.normalized_url + + +# ---- IDN (퓨니코드) --------------------------------------------------------- + + +class TestIDN: + def test_unicode_to_punycode(self) -> None: + # 유니코드 도메인 → 퓨니코드로 통일 (외부 위협 DB 호환) + result = normalize_url("https://☃.com/") + assert "xn--" in result.normalized_url + + def test_punycode_stays_punycode(self) -> None: + # 이미 퓨니코드이면 그대로 유지 + result = normalize_url("https://xn--n3h.com/") + assert "xn--n3h.com" in result.normalized_url + + def test_ascii_domain_unchanged(self) -> None: + result = normalize_url("https://example.com/") + assert result.normalized_url == "https://example.com/" + + +# ---- 통합 ------------------------------------------------------------------- + + +class TestIntegration: + def test_full_normalization(self) -> None: + raw = " HTTP://EXAMPLE.COM:80/a/../b/./c?q=%7e#frag " + result = normalize_url(raw) + assert result.original_url == "HTTP://EXAMPLE.COM:80/a/../b/./c?q=%7e#frag" + assert result.normalized_url == "http://example.com/b/c?q=~" + + def test_preserves_meaningful_query(self) -> None: + result = normalize_url("https://example.com/search?q=hello&lang=ko") + assert "q=hello" in result.normalized_url + assert "lang=ko" in result.normalized_url From c67471238e072b3f51ac9ea58fe02b4c8260e075 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 19:40:35 +0900 Subject: [PATCH 009/173] =?UTF-8?q?[Docs]=20README=20=EC=97=85=EB=8D=B0?= =?UTF-8?q?=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0cf0009..0218eb7 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,13 @@ linclean-fastapi/ │ │ └── __init__.py # Alembic autogen 용 import 모음 │ │ │ ├── schemas/ # Pydantic DTO -│ │ └── common.py # HealthResponse, ErrorResponse 등 +│ │ ├── common.py # HealthResponse, ErrorResponse 등 +│ │ └── analysis.py # NormalizeResult 등 파이프라인 단계별 DTO │ │ -│ └── services/ # 도메인/비즈니스 로직 -│ (4단계 파이프라인 / 외부 연동 / Spring 콜백이 들어갈 자리) +│ └── services/ # 도메인/비즈니스 로직 (파이프라인 단계별 모듈) +│ └── normalizer/ # 1단계: URL 정규화(Canonicalization) +│ ├── __init__.py # 진입점 (normalize_url re-export) +│ └── normalize.py # 입력 검증, 스킴·호스트 정규화, 포트·프래그먼트 제거, 퍼센트 인코딩 정돈, 경로 정규화, IDN 디코딩 │ ├── alembic/ # 외부 피드 캐시 스키마 마이그레이션 │ ├── env.py # SQLite + batch mode 설정 @@ -72,8 +75,11 @@ linclean-fastapi/ │ └── versions/ │ ├── tests/ # pytest 테스트 -│ ├── conftest.py # ASGI AsyncClient + 인메모리 SQLite -│ └── test_health.py +│ ├── conftest.py # 공용 픽스처 +│ ├── demo_normalize.py # URL 정규화 데모 스크립트 +│ └── services/ +│ └── normalizer/ +│ └── test_normalize.py # URL 정규화 단위 테스트 (38개) │ ├── docs/ # 기획서 등 설계 문서 │ └── LinClean_기능_초안.pdf @@ -102,9 +108,13 @@ linclean-fastapi/ `app/models/__init__.py` 에서 import 해야 Alembic autogenerate 가 인식합니다. - **`schemas/`** — 요청·응답 Pydantic 모델. Spring 으로 보내는 분석 결과 DTO (`AnalysisResultCallback`) 도 여기서 정의합니다. -- **`services/`** — 4단계 파이프라인, 외부 API 클라이언트, 점수 산출, Spring - 콜백 호출, AI 정적 분석 호출 등 모든 비즈니스 로직. `Request` 같은 FastAPI +- **`services/`** — 4단계 파이프라인을 **단계별 하위 패키지**로 분리합니다. + 각 패키지의 `__init__.py` 가 해당 단계의 public 진입점을 re-export 하며, + 오케스트레이터(`pipeline.py`)가 이를 순차 호출합니다. `Request` 같은 FastAPI 객체를 받지 않고 `AsyncSession` / 순수 인자만 받습니다. + - **`normalizer/`** — 1단계. `normalize_url()` 로 URL 을 canonical form 으로 + 정규화합니다 (스킴·호스트 소문자화, 기본 포트 제거, 퍼센트 인코딩 정돈, + 경로 dot-segment 해소, IDN 디코딩, 프래그먼트 제거, 입력 검증). - **`middleware/`** — `RequestContextMiddleware` 가 매 요청마다 `X-Request-ID` 를 생성/전파하고 structlog contextvars 에 바인딩합니다. 응답 헤더로도 echo 되며 모든 로그 라인에 자동으로 따라붙습니다. From d06f851188d799d46b74cd651f6dbaa3d2d17809 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 20:16:06 +0900 Subject: [PATCH 010/173] =?UTF-8?q?[Fix]=20URL=20=EA=B8=B8=EC=9D=B4=20?= =?UTF-8?q?=EC=A0=9C=ED=95=9C=202048=20->=201024=EB=A1=9C=20=EB=B3=80?= =?UTF-8?q?=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 2 +- app/services/normalizer/normalize.py | 8 +++----- tests/demo_normalize.py | 2 +- tests/services/normalizer/test_normalize.py | 10 +++++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/app/core/config.py b/app/core/config.py index aa58a9d..520f150 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -76,7 +76,7 @@ def alembic_database_url(self) -> str: rdap_new_domain_threshold_days: int = 30 # ---- Normalizer -------------------------------------------------------- - normalizer_max_url_length: int = 2048 + normalizer_max_url_length: int = 1024 # ---- Scoring ----------------------------------------------------------- score_weight_gsb: int = 50 diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index d82bc0c..9527d68 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -1,7 +1,7 @@ """URL 정규화(Canonicalization) 모듈. 파이프라인 1단계 — 후속 단계(위협 DB 대조·휴리스틱·콘텐츠 분석)가 -동일한 기준으로 URL을 비교할 수 있도록 canonical form으로 변환합니다. +동일한 기준으로 URL을 비교할 수 있도록 canonical form 으로 변환 수행 항목: - 입력 검증 (빈 문자열, 공백 trim, 제어 문자 제거, 최대 길이 제한) @@ -42,7 +42,7 @@ def normalize_url(raw_url: str) -> NormalizeResult: - """원본 URL을 canonical form으로 정규화합니다. + """원본 URL을 canonical form으로 정규화 합니다. Returns: NormalizeResult: original_url(trim 후 원본)과 normalized_url. @@ -88,9 +88,7 @@ def normalize_url(raw_url: str) -> NormalizeResult: # ---- userinfo 제거 (보안) ----------------------------------------------- # RFC 3986 §3.2.1의 userinfo (user:pass@)를 의도적으로 삭제합니다. - # 사유: "https://google.com@evil.com/" 처럼 합법 도메인으로 위장하는 - # 피싱 트릭에 사용되며, 브라우저도 userinfo를 무시하거나 경고합니다. - # parsed.hostname이 실제 호스트(evil.com)를 정확히 반환하므로 안전합니다. + # parsed.hostname이 실제 호스트(evil.com)를 정확히 반환합니다. # ---- 기본 포트 제거 ------------------------------------------------------ port = parsed.port diff --git a/tests/demo_normalize.py b/tests/demo_normalize.py index 1c66dbf..28d3f2b 100644 --- a/tests/demo_normalize.py +++ b/tests/demo_normalize.py @@ -36,7 +36,7 @@ ("공백만", " "), ("지원하지 않는 스킴", "ftp2://example.com"), ("호스트 없음", "https://"), - ("최대 길이 초과", "https://example.com/" + "a" * 2048), + ("최대 길이 초과", "https://example.com/" + "a" * 1024), ] diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py index ad18e2c..24b62cb 100644 --- a/tests/services/normalizer/test_normalize.py +++ b/tests/services/normalizer/test_normalize.py @@ -28,22 +28,22 @@ def test_control_characters_removed(self) -> None: assert "example.com" in result.normalized_url def test_max_length_exceeded_raises(self) -> None: - long_url = "https://example.com/" + "a" * 2048 + long_url = "https://example.com/" + "a" * 1024 with pytest.raises(NormalizationError, match="최대 길이"): normalize_url(long_url) def test_max_length_boundary_passes(self) -> None: - # 정확히 2048자 — 통과해야 함 - padding = 2048 - len("https://example.com/") + # 정확히 1024자 — 통과해야 함 + padding = 1024 - len("https://example.com/") url = "https://example.com/" + "a" * padding result = normalize_url(url) assert result.normalized_url.startswith("https://example.com/") def test_max_length_checked_after_scheme_prepend(self) -> None: # 스킴 없이 입력하면 "https://" 가 붙으므로, 보정 후 길이로 검사해야 함 - padding = 2048 - len("https://") - len("example.com/") + padding = 1024 - len("https://") - len("example.com/") url_without_scheme = "example.com/" + "a" * padding - # 스킴 보정 후 정확히 2048자 → 통과 + # 스킴 보정 후 정확히 1024자 → 통과 result = normalize_url(url_without_scheme) assert result.normalized_url.startswith("https://example.com/") From ee46e4e7b3a4258f0a0780ffb48704d41cfa1444 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 20:28:16 +0900 Subject: [PATCH 011/173] =?UTF-8?q?[Chore]=20=EC=A3=BC=EC=84=9D=20?= =?UTF-8?q?=EC=8A=A4=ED=83=80=EC=9D=BC=20=EB=B3=80=EA=B2=BD=20(What=20->?= =?UTF-8?q?=20Why)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 21 +++-- app/core/logging.py | 6 +- app/db/base.py | 2 +- app/db/session.py | 18 +--- app/middleware/request_context.py | 7 +- app/schemas/analysis.py | 6 +- app/services/normalizer/normalize.py | 94 +++------------------ tests/demo_normalize.py | 76 ++++++++--------- tests/services/normalizer/test_normalize.py | 43 +--------- 9 files changed, 70 insertions(+), 203 deletions(-) diff --git a/app/core/config.py b/app/core/config.py index 520f150..1141dc0 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -16,24 +16,23 @@ class Settings(BaseSettings): case_sensitive=False, ) - # ---- App --------------------------------------------------------------- + # App app_name: str = "LinClean Open API" app_version: str = "0.1.0" environment: Literal["local", "dev", "staging", "production"] = "local" debug: bool = False api_v1_prefix: str = "/api/v1" - # ---- Server ------------------------------------------------------------ + # Server host: str = "0.0.0.0" # noqa: S104 port: int = 8000 log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" log_json: bool = False - # ---- CORS -------------------------------------------------------------- + # CORS cors_origins: list[str] = Field(default_factory=lambda: ["http://localhost:3000"]) - # ---- Database (SQLite) ------------------------------------------------- - # Path is resolved relative to the project root unless absolute. + # Database — path resolved relative to project root unless absolute. sqlite_path: str = "data/linclean.db" db_echo: bool = False @@ -51,10 +50,10 @@ def database_url(self) -> str: @computed_field # type: ignore[prop-decorator] @property def alembic_database_url(self) -> str: - # Alembic runs migrations synchronously — use the stdlib sqlite driver. + # Alembic runs synchronously — needs the stdlib sqlite driver. return f"sqlite:///{self.sqlite_file}" - # ---- External: Google Safe Browsing ------------------------------------ + # Google Safe Browsing gsb_api_key: str | None = None gsb_api_url: str = ( "https://safebrowsing.googleapis.com/v4/threatMatches:find" @@ -63,22 +62,22 @@ def alembic_database_url(self) -> str: gsb_client_version: str = "0.1.0" gsb_timeout_seconds: float = 5.0 - # ---- External: URLhaus ------------------------------------------------- + # URLhaus urlhaus_recent_csv_url: str = "https://urlhaus.abuse.ch/downloads/csv_recent/" urlhaus_online_csv_url: str = "https://urlhaus.abuse.ch/downloads/csv_online/" urlhaus_refresh_interval_seconds: int = 60 * 60 # 60 minutes urlhaus_download_timeout_seconds: float = 60.0 - # ---- External: RDAP ---------------------------------------------------- + # RDAP rdap_bootstrap_url: str = "https://rdap.org/domain/" rdap_timeout_seconds: float = 5.0 rdap_cache_ttl_seconds: int = 60 * 60 * 24 # 24h rdap_new_domain_threshold_days: int = 30 - # ---- Normalizer -------------------------------------------------------- + # Normalizer normalizer_max_url_length: int = 1024 - # ---- Scoring ----------------------------------------------------------- + # Scoring score_weight_gsb: int = 50 score_weight_urlhaus: int = 50 score_weight_rdap: int = 20 diff --git a/app/core/logging.py b/app/core/logging.py index 7891d59..5868f59 100644 --- a/app/core/logging.py +++ b/app/core/logging.py @@ -16,11 +16,7 @@ def _drop_color_message_key(_: Any, __: str, event_dict: EventDict) -> EventDict def configure_logging() -> None: - """Configure structlog + stdlib logging. - - All logs (including from uvicorn, sqlalchemy, etc.) are routed through - structlog so we get consistent JSON or pretty output everywhere. - """ + """Wire structlog as the single output path for all loggers.""" timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True) shared_processors: list[Processor] = [ diff --git a/app/db/base.py b/app/db/base.py index c0a1213..d59fe9d 100644 --- a/app/db/base.py +++ b/app/db/base.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.sql import func -# Naming convention for constraints — keeps Alembic autogenerate stable. +# Explicit naming convention keeps Alembic autogenerate deterministic. NAMING_CONVENTION = { "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", diff --git a/app/db/session.py b/app/db/session.py index 7c359b4..c3eb0b8 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -12,13 +12,8 @@ from app.core.config import settings -# SQLite-specific engine. Notes: -# - check_same_thread=False is required because aiosqlite hands the -# connection across threads internally. -# - We don't tune pool size: SQLite is single-writer, and the default -# pool is fine for an async app of this scale. -# - We ensure the parent directory exists at import time so the very -# first connection doesn't fail in a fresh checkout. +# aiosqlite requires check_same_thread=False (it passes connections across threads). +# Ensure data dir exists so a fresh checkout doesn't fail on first connect. settings.sqlite_file.parent.mkdir(parents=True, exist_ok=True) engine: AsyncEngine = create_async_engine( @@ -31,12 +26,7 @@ @event.listens_for(Engine, "connect") def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: - """Apply sane defaults for every SQLite connection. - - - WAL gives concurrent readers while a writer is active. - - foreign_keys=ON is off by default in SQLite (!) and must be re-enabled. - - NORMAL synchronous trades a tiny crash-recovery window for big writes. - """ + """WAL for concurrent reads, foreign_keys=ON (off by default in SQLite).""" cursor = dbapi_connection.cursor() try: cursor.execute("PRAGMA journal_mode=WAL") @@ -55,7 +45,7 @@ def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: async def get_db() -> AsyncIterator[AsyncSession]: - """FastAPI dependency yielding an AsyncSession with rollback-on-error.""" + """Yield an AsyncSession; rolls back on unhandled errors.""" async with SessionLocal() as session: try: yield session diff --git a/app/middleware/request_context.py b/app/middleware/request_context.py index 40b6e96..4f5679f 100644 --- a/app/middleware/request_context.py +++ b/app/middleware/request_context.py @@ -12,12 +12,7 @@ class RequestContextMiddleware(BaseHTTPMiddleware): - """Bind a request_id + access log entry to every request. - - - Reuses an inbound `X-Request-ID` if present, else generates one. - - Binds it into structlog contextvars so all logs in the request carry it. - - Emits a single structured access log line per request. - """ + """Assign a request ID and emit a structured access log per request.""" async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 951bd3c..0da747c 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -2,7 +2,7 @@ class NormalizeResult(BaseModel): - """1단계 — URL 정규화(Canonicalization) 결과.""" + """Stage 1 — URL canonicalization result.""" - original_url: str = Field(description="입력 원본 URL (trim 후)") - normalized_url: str = Field(description="정규화된 URL") + original_url: str = Field(description="Original URL after trimming") + normalized_url: str = Field(description="Canonicalized URL") diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index 9527d68..ac74d10 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -1,19 +1,7 @@ -"""URL 정규화(Canonicalization) 모듈. - -파이프라인 1단계 — 후속 단계(위협 DB 대조·휴리스틱·콘텐츠 분석)가 -동일한 기준으로 URL을 비교할 수 있도록 canonical form 으로 변환 - -수행 항목: - - 입력 검증 (빈 문자열, 공백 trim, 제어 문자 제거, 최대 길이 제한) - - 스킴 검증 / 기본값 보정 (http, https, ftp만 허용) - - 스킴·호스트 소문자화 (경로는 대소문자 보존) - - userinfo 제거 (피싱 방지 — google.com@evil.com 같은 트릭 차단) - - 기본 포트 제거 (:80, :443, :21) - - 프래그먼트 제거 - - 퍼센트 인코딩 정돈 (unreserved 디코딩, reserved 대문자 hex 통일) - - 경로 정규화 (빈 경로 → "/", dot-segment 해소, 연속 슬래시 축소) - - IDN 정규화 (유니코드 → 퓨니코드 통일, 외부 위협 DB 호환) - - 원본 보존 (NormalizeResult.original_url) +"""URL canonicalization — pipeline stage 1. + +Produces a canonical form so downstream stages (threat DB lookup, +heuristics, content analysis) can compare URLs consistently. """ from __future__ import annotations @@ -29,42 +17,28 @@ _DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443, "ftp": 21} _ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https", "ftp"}) -# RFC 3986 §2.3 — unreserved characters +# RFC 3986 §2.3 _UNRESERVED: frozenset[str] = frozenset( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" ) -# 제어 문자 패턴 (C0, C1, DEL + 유니코드 제어 카테고리) _CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f-\x9f]") - -# percent-encoded triplet 패턴 _PCT_ENCODED_RE = re.compile(r"%([0-9A-Fa-f]{2})") def normalize_url(raw_url: str) -> NormalizeResult: - """원본 URL을 canonical form으로 정규화 합니다. - - Returns: - NormalizeResult: original_url(trim 후 원본)과 normalized_url. - - Raises: - NormalizationError: 검증 실패 시. - """ - # ---- 입력 검증 ---------------------------------------------------------- + """Return a canonical NormalizeResult, or raise NormalizationError.""" original = raw_url.strip() if not original: raise NormalizationError(message="빈 URL입니다.") - # 제어 문자 제거 cleaned = _CONTROL_CHAR_RE.sub("", original) - # NFC 정규화 (유니코드 합성) cleaned = unicodedata.normalize("NFC", cleaned) - # ---- 스킴 보정 (길이 검사보다 먼저 수행) --------------------------------- + # Scheme must be resolved before length check — prepending adds bytes. if "://" not in cleaned: cleaned = "https://" + cleaned - # ---- 최대 길이 검사 (스킴 보정 후) --------------------------------------- max_len = settings.normalizer_max_url_length if len(cleaned) > max_len: raise NormalizationError( @@ -80,49 +54,28 @@ def normalize_url(raw_url: str) -> NormalizeResult: if scheme not in _ALLOWED_SCHEMES: raise NormalizationError(message=f"지원하지 않는 스킴: {scheme}") - # ---- 호스트 소문자화 + IDN 정규화 ---------------------------------------- hostname = parsed.hostname or "" hostname = _normalize_idn(hostname.lower()) if not hostname: raise NormalizationError(message="호스트가 비어있습니다.") - # ---- userinfo 제거 (보안) ----------------------------------------------- - # RFC 3986 §3.2.1의 userinfo (user:pass@)를 의도적으로 삭제합니다. - # parsed.hostname이 실제 호스트(evil.com)를 정확히 반환합니다. - - # ---- 기본 포트 제거 ------------------------------------------------------ + # Reassembling netloc from parsed.hostname implicitly drops userinfo + # (RFC 3986 §3.2.1), which is abused for phishing (google.com@evil.com). port = parsed.port if port and port == _DEFAULT_PORTS.get(scheme): port = None netloc = hostname if port is None else f"{hostname}:{port}" - # ---- 경로 정규화 (대소문자 보존) ----------------------------------------- path = _normalize_path(parsed.path) - - # ---- params: 퍼센트 인코딩 정돈 ----------------------------------------- params = _normalize_pct_encoding(parsed.params) - - # ---- 쿼리: 퍼센트 인코딩 정돈 ------------------------------------------- query = _normalize_pct_encoding(parsed.query) - # ---- 프래그먼트 제거 ----------------------------------------------------- normalized = urlunparse((scheme, netloc, path, params, query, "")) - return NormalizeResult(original_url=original, normalized_url=normalized) -# --------------------------------------------------------------------------- -# 내부 헬퍼 -# --------------------------------------------------------------------------- - - def _normalize_idn(hostname: str) -> str: - """호스트를 퓨니코드(ASCII-compatible encoding)로 통일합니다. - - GSB·URLhaus 등 외부 위협 DB가 퓨니코드 형태로 URL을 저장하므로, - 유니코드 도메인을 퓨니코드로 변환해야 매칭이 보장됩니다. - 이미 퓨니코드(ASCII)이면 그대로 반환합니다. - """ + """Convert unicode hostname to punycode for threat DB compatibility.""" try: return hostname.encode("idna").decode("ascii") except (UnicodeError, UnicodeDecodeError): @@ -130,31 +83,19 @@ def _normalize_idn(hostname: str) -> str: def _normalize_path(raw_path: str) -> str: - """경로를 정규화합니다. - - - 빈 경로 → "/" - - 퍼센트 인코딩 정돈 - - dot-segment 해소 (RFC 3986 §5.2.4) - - 연속 슬래시 축소 - """ path = raw_path or "/" - - # 퍼센트 인코딩 정돈 (경로는 대소문자 보존하므로 safe에 슬래시 포함) path = _normalize_pct_encoding(path) - # 연속 슬래시 축소 — 보안 목적의 의도적 정규화. - # RFC 상 //a와 /a는 다른 리소스이지만, 공격자가 //를 삽입해 - # 위협 DB 패턴 매칭을 우회하는 것을 방지합니다. + # RFC treats //a and /a as different resources, but attackers insert + # extra slashes to dodge threat DB pattern matching. path = re.sub(r"/{2,}", "/", path) - # dot-segment 해소 path = _resolve_dot_segments(path) - return path or "/" def _resolve_dot_segments(path: str) -> str: - """RFC 3986 §5.2.4 — dot-segment를 해소합니다.""" + """RFC 3986 §5.2.4""" segments = path.split("/") resolved: list[str] = [] for seg in segments: @@ -165,28 +106,21 @@ def _resolve_dot_segments(path: str) -> str: resolved.pop() else: resolved.append(seg) - # 원래 절대 경로면 선행 빈 문자열 보장 if path.startswith("/") and (not resolved or resolved[0] != ""): resolved.insert(0, "") return "/".join(resolved) def _normalize_pct_encoding(value: str) -> str: - """퍼센트 인코딩을 정돈합니다. - - - unreserved 문자(%41 → A): 디코딩 - - reserved / 기타 문자: hex를 대문자로 통일 (%2f → %2F) - """ + """Decode unreserved (%41->A), uppercase reserved hex (%2f->%2F).""" if not value: return value def _replace(match: re.Match[str]) -> str: hex_str = match.group(1) char = chr(int(hex_str, 16)) - # unreserved면 디코딩 if char in _UNRESERVED: return char - # 그 외는 대문자 hex 통일 return f"%{hex_str.upper()}" return _PCT_ENCODED_RE.sub(_replace, value) diff --git a/tests/demo_normalize.py b/tests/demo_normalize.py index 28d3f2b..b1038bb 100644 --- a/tests/demo_normalize.py +++ b/tests/demo_normalize.py @@ -1,4 +1,4 @@ -"""URL 정규화 데모 +"""URL normalization demo. Usage: python -m tests.demo_normalize @@ -9,61 +9,55 @@ from app.core.exceptions import NormalizationError from app.services.normalizer import normalize_url -DIVIDER = "─" * 72 +DIVIDER = "-" * 72 CASES: list[tuple[str, str]] = [ - # (설명, 입력 URL) - ("스킴·호스트 소문자화 + 경로 대소문자 보존", "HTTP://EXAMPLE.COM/Path/To/Page"), - ("기본 포트 제거 (80, 443)", "https://example.com:443/secure"), - ("비기본 포트 유지", "https://example.com:8080/api"), - ("프래그먼트 제거", "https://example.com/page#section-2"), - ("스킴 없으면 https 보정", "example.com/hello"), - ("퍼센트 인코딩: unreserved 디코딩 (%7E→~)", "https://example.com/%7Euser/%41%42"), - ("퍼센트 인코딩: reserved 대문자 hex 통일 (%2f→%2F)", "https://example.com/a%2fb%3fc"), - ("경로 dot-segment 해소 (/../)", "https://example.com/a/b/../c/./d"), - ("연속 슬래시 축소 (보안: DB 매칭 우회 방지)", "https://example.com//a///b"), - ("userinfo 제거 (피싱: google.com@evil.com)", "https://google.com@evil.com/login"), - ("params 퍼센트 인코딩 정돈", "https://example.com/path;%7eparam?q=1"), - ("IDN 유니코드 → 퓨니코드 통일 (외부 DB 호환)", "https://☃.com/path"), - ("IDN 퓨니코드 유지", "https://xn--n3h.com/path"), - ("제어 문자 제거", "https://exam\x00ple.com/pa\x01th"), - ("공백 trim", " https://example.com/page "), - ("종합", " HTTP://EXAMPLE.COM:80/a/b/../c/./d?q=%7e&x=%2f#top "), + ("Scheme/host lowercasing, path case preserved", "HTTP://EXAMPLE.COM/Path/To/Page"), + ("Default port removal (80, 443)", "https://example.com:443/secure"), + ("Non-default port kept", "https://example.com:8080/api"), + ("Fragment removal", "https://example.com/page#section-2"), + ("Missing scheme defaults to https", "example.com/hello"), + ("Unreserved pct-decode (%7E->~)", "https://example.com/%7Euser/%41%42"), + ("Reserved hex uppercased (%2f->%2F)", "https://example.com/a%2fb%3fc"), + ("Dot-segment resolution (/../)", "https://example.com/a/b/../c/./d"), + ("Consecutive slash collapse", "https://example.com//a///b"), + ("Userinfo stripped (phishing vector)", "https://google.com@evil.com/login"), + ("Params pct-encoding normalized", "https://example.com/path;%7eparam?q=1"), + ("IDN unicode -> punycode", "https://xn--n3h.com/path"), + ("IDN punycode passthrough", "https://xn--n3h.com/path"), + ("Control character removal", "https://exam\x00ple.com/pa\x01th"), + ("Whitespace trimming", " https://example.com/page "), + ("Combined", " HTTP://EXAMPLE.COM:80/a/b/../c/./d?q=%7e&x=%2f#top "), ] ERROR_CASES: list[tuple[str, str]] = [ - ("빈 문자열", ""), - ("공백만", " "), - ("지원하지 않는 스킴", "ftp2://example.com"), - ("호스트 없음", "https://"), - ("최대 길이 초과", "https://example.com/" + "a" * 1024), + ("Empty string", ""), + ("Whitespace only", " "), + ("Unsupported scheme", "ftp2://example.com"), + ("No host", "https://"), + ("Exceeds max length", "https://example.com/" + "a" * 1024), ] def main() -> None: print() - print("╔══════════════════════════════════════════════════════════════════════╗") - print("║ LinClean — URL Normalization Test ║") - print("╚══════════════════════════════════════════════════════════════════════╝") + print("=" * 72) + print(" LinClean -- URL Normalization Demo") + print("=" * 72) - # ── 정상 케이스 ────────────────────────────────────────────────────────── - print() - print("■ 정상 케이스") + print("\n[Success cases]") print(DIVIDER) for i, (desc, raw) in enumerate(CASES, 1): result = normalize_url(raw) - # 제어 문자가 포함된 입력은 repr로 표시 display_input = repr(raw) if any(ord(c) < 32 for c in raw) else raw print(f" [{i:02d}] {desc}") - print(f" 입력 → {display_input}") - print(f" 원본 → {result.original_url}") - print(f" 정규화 → {result.normalized_url}") + print(f" input : {display_input}") + print(f" original : {result.original_url}") + print(f" normalized : {result.normalized_url}") print(DIVIDER) - # ── 에러 케이스 ────────────────────────────────────────────────────────── - print() - print("■ 에러 케이스 (NormalizationError)") + print("\n[Error cases (NormalizationError)]") print(DIVIDER) for i, (desc, raw) in enumerate(ERROR_CASES, 1): @@ -71,12 +65,12 @@ def main() -> None: try: normalize_url(raw) print(f" [{i:02d}] {desc}") - print(f" 입력 → {display_input}") - print(f" 결과 → ⚠ 예외가 발생하지 않음!") + print(f" input : {display_input}") + print(f" result : WARN -- expected error not raised!") except NormalizationError as e: print(f" [{i:02d}] {desc}") - print(f" 입력 → {display_input}") - print(f" 에러 → {e.message}") + print(f" input : {display_input}") + print(f" error : {e.message}") print(DIVIDER) print() diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py index 24b62cb..d688ae2 100644 --- a/tests/services/normalizer/test_normalize.py +++ b/tests/services/normalizer/test_normalize.py @@ -1,11 +1,9 @@ -"""normalize_url 단위 테스트.""" +"""normalize_url unit tests.""" import pytest from app.core.exceptions import NormalizationError from app.services.normalizer.normalize import normalize_url -# ---- 입력 검증 --------------------------------------------------------------- - class TestInputValidation: def test_empty_string_raises(self) -> None: @@ -33,24 +31,18 @@ def test_max_length_exceeded_raises(self) -> None: normalize_url(long_url) def test_max_length_boundary_passes(self) -> None: - # 정확히 1024자 — 통과해야 함 padding = 1024 - len("https://example.com/") url = "https://example.com/" + "a" * padding result = normalize_url(url) assert result.normalized_url.startswith("https://example.com/") def test_max_length_checked_after_scheme_prepend(self) -> None: - # 스킴 없이 입력하면 "https://" 가 붙으므로, 보정 후 길이로 검사해야 함 padding = 1024 - len("https://") - len("example.com/") url_without_scheme = "example.com/" + "a" * padding - # 스킴 보정 후 정확히 1024자 → 통과 result = normalize_url(url_without_scheme) assert result.normalized_url.startswith("https://example.com/") -# ---- 원본 보존 --------------------------------------------------------------- - - class TestOriginalPreservation: def test_original_url_preserved_after_trim(self) -> None: result = normalize_url(" HTTP://EXAMPLE.COM/Path ") @@ -62,9 +54,6 @@ def test_original_differs_from_normalized(self) -> None: assert result.normalized_url == "http://example.com/Path" -# ---- 스킴 ------------------------------------------------------------------- - - class TestScheme: def test_lowercase_scheme(self) -> None: result = normalize_url("HTTP://Example.com") @@ -87,9 +76,6 @@ def test_ftp_allowed(self) -> None: assert result.normalized_url.startswith("ftp://") -# ---- 호스트 ----------------------------------------------------------------- - - class TestHost: def test_lowercase_host(self) -> None: result = normalize_url("https://EXAMPLE.COM/") @@ -100,7 +86,6 @@ def test_scheme_only_raises(self) -> None: normalize_url("https://") def test_userinfo_stripped_for_phishing(self) -> None: - # google.com@evil.com → 실제 호스트는 evil.com result = normalize_url("https://google.com@evil.com/path") assert "evil.com" in result.normalized_url assert "google.com@" not in result.normalized_url @@ -111,9 +96,6 @@ def test_basic_auth_userinfo_stripped(self) -> None: assert "example.com/api" in result.normalized_url -# ---- 포트 ------------------------------------------------------------------- - - class TestPort: def test_default_http_port_removed(self) -> None: result = normalize_url("http://example.com:80/path") @@ -132,9 +114,6 @@ def test_non_default_port_kept(self) -> None: assert ":8080" in result.normalized_url -# ---- 경로 ------------------------------------------------------------------- - - class TestPath: def test_empty_path_becomes_slash(self) -> None: result = normalize_url("https://example.com") @@ -145,7 +124,6 @@ def test_path_case_preserved(self) -> None: assert "/Path/To/Page" in result.normalized_url def test_double_slash_collapsed(self) -> None: - # 보안: 공격자가 //를 삽입해 위협 DB 패턴 매칭을 우회하는 것을 방지 result = normalize_url("https://example.com//a//b") assert "//" not in result.normalized_url.split("://", 1)[1] @@ -166,9 +144,6 @@ def test_complex_dot_segments(self) -> None: assert result.normalized_url == "https://example.com/a/d" -# ---- 프래그먼트 ------------------------------------------------------------- - - class TestFragment: def test_fragment_removed(self) -> None: result = normalize_url("https://example.com/page#section") @@ -180,22 +155,16 @@ def test_fragment_with_query(self) -> None: assert "q=1" in result.normalized_url -# ---- 퍼센트 인코딩 ---------------------------------------------------------- - - class TestPercentEncoding: def test_unreserved_decoded(self) -> None: - # %41 = 'A' (unreserved) → 디코딩 result = normalize_url("https://example.com/%41%42%43") assert "/ABC" in result.normalized_url def test_tilde_decoded(self) -> None: - # %7E = '~' (unreserved) result = normalize_url("https://example.com/%7Euser") assert "/~user" in result.normalized_url def test_reserved_hex_uppercased(self) -> None: - # %2f = '/' (reserved) → %2F 대문자 통일 result = normalize_url("https://example.com/path%2fmore") assert "%2F" in result.normalized_url @@ -204,7 +173,6 @@ def test_already_uppercase_hex_unchanged(self) -> None: assert "%2F" in result.normalized_url def test_space_encoding_uppercased(self) -> None: - # %20 = ' ' (not unreserved) → %20 유지 (이미 대문자) result = normalize_url("https://example.com/my%20page") assert "%20" in result.normalized_url @@ -213,22 +181,16 @@ def test_query_encoding_normalized(self) -> None: assert "q=abc" in result.normalized_url def test_params_encoding_normalized(self) -> None: - # path params(;)도 path, query와 동일하게 퍼센트 인코딩 정돈 result = normalize_url("https://example.com/path;%7eparam?q=1") assert ";~param" in result.normalized_url -# ---- IDN (퓨니코드) --------------------------------------------------------- - - class TestIDN: def test_unicode_to_punycode(self) -> None: - # 유니코드 도메인 → 퓨니코드로 통일 (외부 위협 DB 호환) result = normalize_url("https://☃.com/") assert "xn--" in result.normalized_url def test_punycode_stays_punycode(self) -> None: - # 이미 퓨니코드이면 그대로 유지 result = normalize_url("https://xn--n3h.com/") assert "xn--n3h.com" in result.normalized_url @@ -237,9 +199,6 @@ def test_ascii_domain_unchanged(self) -> None: assert result.normalized_url == "https://example.com/" -# ---- 통합 ------------------------------------------------------------------- - - class TestIntegration: def test_full_normalization(self) -> None: raw = " HTTP://EXAMPLE.COM:80/a/../b/./c?q=%7e#frag " From bc4a3c699f4a2f571aef932efaae465d60ded634 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 21:51:31 +0900 Subject: [PATCH 012/173] =?UTF-8?q?[Fix]=20parsed.port=20=EC=A0=91?= =?UTF-8?q?=EA=B7=BC=20=EC=8B=9C=20ValueError=20=EB=AF=B8=EC=B2=98?= =?UTF-8?q?=EB=A6=AC=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/normalize.py | 5 ++++- tests/services/normalizer/test_normalize.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index ac74d10..15a8e2a 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -61,7 +61,10 @@ def normalize_url(raw_url: str) -> NormalizeResult: # Reassembling netloc from parsed.hostname implicitly drops userinfo # (RFC 3986 §3.2.1), which is abused for phishing (google.com@evil.com). - port = parsed.port + try: + port = parsed.port + except ValueError as e: + raise NormalizationError(message=f"유효하지 않은 포트: {e}") from e if port and port == _DEFAULT_PORTS.get(scheme): port = None netloc = hostname if port is None else f"{hostname}:{port}" diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py index d688ae2..17760cb 100644 --- a/tests/services/normalizer/test_normalize.py +++ b/tests/services/normalizer/test_normalize.py @@ -113,6 +113,14 @@ def test_non_default_port_kept(self) -> None: result = normalize_url("https://example.com:8080/") assert ":8080" in result.normalized_url + def test_out_of_range_port_raises(self) -> None: + with pytest.raises(NormalizationError, match="유효하지 않은 포트"): + normalize_url("https://example.com:99999/") + + def test_non_numeric_port_raises(self) -> None: + with pytest.raises(NormalizationError, match="유효하지 않은 포트"): + normalize_url("example.com:notaport/foo") + class TestPath: def test_empty_path_becomes_slash(self) -> None: From 448e22be902f0b8e10db28c6a36589b3f0701a10 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 22:12:08 +0900 Subject: [PATCH 013/173] =?UTF-8?q?[Fix]=20IPv6=20=EC=A3=BC=EC=86=8C=20?= =?UTF-8?q?=EC=9E=85=EB=A0=A5=20=EC=8B=9C=20netloc=20=EC=9E=AC=EC=A1=B0?= =?UTF-8?q?=EB=A6=BD=20=EC=98=A4=EB=A5=98=20=EB=B0=A9=EC=A7=80(IPv6=20?= =?UTF-8?q?=EB=AF=B8=EC=A7=80=EC=9B=90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/normalize.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index 15a8e2a..9a873e5 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -55,6 +55,8 @@ def normalize_url(raw_url: str) -> NormalizeResult: raise NormalizationError(message=f"지원하지 않는 스킴: {scheme}") hostname = parsed.hostname or "" + if ":" in hostname: + raise NormalizationError(message="IPv6 주소는 지원하지 않습니다.") hostname = _normalize_idn(hostname.lower()) if not hostname: raise NormalizationError(message="호스트가 비어있습니다.") From eaba0a9ef68805fc954579cbe4a3215e839a8ca4 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 22:20:09 +0900 Subject: [PATCH 014/173] =?UTF-8?q?[Fix]=20=5Fnormalize=5Fidn=20=EC=A4=91?= =?UTF-8?q?=EB=B3=B5=20=EC=98=88=EC=99=B8=20=ED=83=80=EC=9E=85=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/normalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index 9a873e5..337a0ff 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -83,7 +83,7 @@ def _normalize_idn(hostname: str) -> str: """Convert unicode hostname to punycode for threat DB compatibility.""" try: return hostname.encode("idna").decode("ascii") - except (UnicodeError, UnicodeDecodeError): + except UnicodeError: return hostname From 6e65c266d4d2e055ebdc364b064e96a99bbd6d10 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 22:26:32 +0900 Subject: [PATCH 015/173] =?UTF-8?q?[Fix]=20=EA=B2=BD=EB=A1=9C=20=EB=82=B4?= =?UTF-8?q?=20://=20=ED=8F=AC=ED=95=A8=20=EC=8B=9C=20=EC=8A=A4=ED=82=B4=20?= =?UTF-8?q?=EA=B0=90=EC=A7=80=20=EC=98=A4=EB=A5=98=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/normalize.py | 3 ++- tests/services/normalizer/test_normalize.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index 337a0ff..ae26793 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -36,7 +36,8 @@ def normalize_url(raw_url: str) -> NormalizeResult: cleaned = unicodedata.normalize("NFC", cleaned) # Scheme must be resolved before length check — prepending adds bytes. - if "://" not in cleaned: + # Simple "://" check misses cases like "example.com/path://weird". + if not re.match(r'^[a-zA-Z][a-zA-Z0-9+\-.]*://', cleaned): cleaned = "https://" + cleaned max_len = settings.normalizer_max_url_length diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py index 17760cb..213c402 100644 --- a/tests/services/normalizer/test_normalize.py +++ b/tests/services/normalizer/test_normalize.py @@ -67,6 +67,10 @@ def test_no_scheme_defaults_to_https(self) -> None: result = normalize_url("example.com/path") assert result.normalized_url == "https://example.com/path" + def test_scheme_like_string_in_path_gets_scheme_prepended(self) -> None: + result = normalize_url("example.com/path://weird") + assert result.normalized_url.startswith("https://example.com/") + def test_unsupported_scheme_raises(self) -> None: with pytest.raises(NormalizationError, match="지원하지 않는 스킴"): normalize_url("ftp2://example.com") From 2edc7eb13dcc33d855a7849b847976d49b8ac385 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Fri, 10 Apr 2026 22:40:09 +0900 Subject: [PATCH 016/173] =?UTF-8?q?[Chore]=20=EC=A0=84=EC=B2=B4=20?= =?UTF-8?q?=EC=A3=BC=EC=84=9D=20=ED=95=9C=EA=B5=AD=EC=96=B4=EB=A1=9C=20?= =?UTF-8?q?=ED=86=B5=EC=9D=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- alembic/env.py | 3 +-- app/core/config.py | 12 ++++----- app/core/logging.py | 4 +-- app/db/base.py | 2 +- app/db/session.py | 8 +++--- app/services/normalizer/normalize.py | 27 ++++++++++----------- tests/demo_normalize.py | 4 +-- tests/services/normalizer/test_normalize.py | 2 +- 8 files changed, 30 insertions(+), 32 deletions(-) diff --git a/alembic/env.py b/alembic/env.py index 7854f21..6f8dd71 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -15,8 +15,7 @@ target_metadata = Base.metadata -# SQLite + Alembic: enable batch mode so ALTER TABLE migrations work, -# since SQLite has very limited native ALTER support. +# SQLite는 ALTER TABLE 지원이 빈약해서 batch 모드 필수. RENDER_AS_BATCH = True diff --git a/app/core/config.py b/app/core/config.py index 1141dc0..d804051 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -16,14 +16,14 @@ class Settings(BaseSettings): case_sensitive=False, ) - # App + # 앱 기본 app_name: str = "LinClean Open API" app_version: str = "0.1.0" environment: Literal["local", "dev", "staging", "production"] = "local" debug: bool = False api_v1_prefix: str = "/api/v1" - # Server + # 서버 host: str = "0.0.0.0" # noqa: S104 port: int = 8000 log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" @@ -32,7 +32,7 @@ class Settings(BaseSettings): # CORS cors_origins: list[str] = Field(default_factory=lambda: ["http://localhost:3000"]) - # Database — path resolved relative to project root unless absolute. + # DB — 절대 경로 아니면 프로젝트 루트 기준으로 해석됨. sqlite_path: str = "data/linclean.db" db_echo: bool = False @@ -50,7 +50,7 @@ def database_url(self) -> str: @computed_field # type: ignore[prop-decorator] @property def alembic_database_url(self) -> str: - # Alembic runs synchronously — needs the stdlib sqlite driver. + # Alembic은 동기 실행이라 stdlib sqlite 드라이버 필요. return f"sqlite:///{self.sqlite_file}" # Google Safe Browsing @@ -74,10 +74,10 @@ def alembic_database_url(self) -> str: rdap_cache_ttl_seconds: int = 60 * 60 * 24 # 24h rdap_new_domain_threshold_days: int = 30 - # Normalizer + # 정규화 normalizer_max_url_length: int = 1024 - # Scoring + # 점수 산정 score_weight_gsb: int = 50 score_weight_urlhaus: int = 50 score_weight_rdap: int = 20 diff --git a/app/core/logging.py b/app/core/logging.py index 5868f59..9e54f7f 100644 --- a/app/core/logging.py +++ b/app/core/logging.py @@ -16,7 +16,7 @@ def _drop_color_message_key(_: Any, __: str, event_dict: EventDict) -> EventDict def configure_logging() -> None: - """Wire structlog as the single output path for all loggers.""" + """structlog를 모든 로거의 단일 출력 경로로 설정.""" timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True) shared_processors: list[Processor] = [ @@ -60,7 +60,7 @@ def configure_logging() -> None: root.handlers = [handler] root.setLevel(settings.log_level) - # Quiet noisy libs; route through root handler + # 로그 시끄러운 라이브러리 억제, root 핸들러로 통합 for name in ("uvicorn", "uvicorn.error", "uvicorn.access"): lg = logging.getLogger(name) lg.handlers = [] diff --git a/app/db/base.py b/app/db/base.py index d59fe9d..e810128 100644 --- a/app/db/base.py +++ b/app/db/base.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.sql import func -# Explicit naming convention keeps Alembic autogenerate deterministic. +# 네이밍 컨벤션 명시 — Alembic autogenerate 결과 일관성 유지용. NAMING_CONVENTION = { "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", diff --git a/app/db/session.py b/app/db/session.py index c3eb0b8..0c3c600 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -12,8 +12,8 @@ from app.core.config import settings -# aiosqlite requires check_same_thread=False (it passes connections across threads). -# Ensure data dir exists so a fresh checkout doesn't fail on first connect. +# aiosqlite는 스레드 간 커넥션 전달하므로 check_same_thread=False 필수. +# 데이터 디렉토리 없으면 첫 연결에서 터지니까 미리 생성. settings.sqlite_file.parent.mkdir(parents=True, exist_ok=True) engine: AsyncEngine = create_async_engine( @@ -26,7 +26,7 @@ @event.listens_for(Engine, "connect") def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: - """WAL for concurrent reads, foreign_keys=ON (off by default in SQLite).""" + """WAL 모드로 동시 읽기 지원, foreign_keys=ON (SQLite 기본값은 OFF).""" cursor = dbapi_connection.cursor() try: cursor.execute("PRAGMA journal_mode=WAL") @@ -45,7 +45,7 @@ def _enable_sqlite_pragmas(dbapi_connection: Any, _: Any) -> None: async def get_db() -> AsyncIterator[AsyncSession]: - """Yield an AsyncSession; rolls back on unhandled errors.""" + """AsyncSession 제공. 미처리 예외 시 롤백.""" async with SessionLocal() as session: try: yield session diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index ae26793..c68bc57 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -1,7 +1,7 @@ -"""URL canonicalization — pipeline stage 1. +"""URL 정규화 — 파이프라인 1단계. -Produces a canonical form so downstream stages (threat DB lookup, -heuristics, content analysis) can compare URLs consistently. +위협 DB 조회, 휴리스틱, 콘텐츠 분석 등 후속 단계에서 +URL을 일관되게 비교할 수 있도록 정규 형태로 변환함. """ from __future__ import annotations @@ -17,7 +17,6 @@ _DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443, "ftp": 21} _ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https", "ftp"}) -# RFC 3986 §2.3 _UNRESERVED: frozenset[str] = frozenset( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" ) @@ -27,7 +26,7 @@ def normalize_url(raw_url: str) -> NormalizeResult: - """Return a canonical NormalizeResult, or raise NormalizationError.""" + """정규화된 NormalizeResult 반환. 실패 시 NormalizationError.""" original = raw_url.strip() if not original: raise NormalizationError(message="빈 URL입니다.") @@ -35,8 +34,8 @@ def normalize_url(raw_url: str) -> NormalizeResult: cleaned = _CONTROL_CHAR_RE.sub("", original) cleaned = unicodedata.normalize("NFC", cleaned) - # Scheme must be resolved before length check — prepending adds bytes. - # Simple "://" check misses cases like "example.com/path://weird". + # 길이 체크 전에 스킴 확정해야 함 — 스킴 붙이면 바이트 늘어남. + # 단순 "://" 체크는 "example.com/path://weird" 같은 케이스 놓침. if not re.match(r'^[a-zA-Z][a-zA-Z0-9+\-.]*://', cleaned): cleaned = "https://" + cleaned @@ -62,8 +61,8 @@ def normalize_url(raw_url: str) -> NormalizeResult: if not hostname: raise NormalizationError(message="호스트가 비어있습니다.") - # Reassembling netloc from parsed.hostname implicitly drops userinfo - # (RFC 3986 §3.2.1), which is abused for phishing (google.com@evil.com). + # parsed.hostname으로 netloc 재조립하면 userinfo 자동 제거됨. + # 피싱에 악용되는 패턴임 (google.com@evil.com). try: port = parsed.port except ValueError as e: @@ -81,7 +80,7 @@ def normalize_url(raw_url: str) -> NormalizeResult: def _normalize_idn(hostname: str) -> str: - """Convert unicode hostname to punycode for threat DB compatibility.""" + """유니코드 호스트명을 punycode로 변환. 위협 DB 매칭 호환용.""" try: return hostname.encode("idna").decode("ascii") except UnicodeError: @@ -92,8 +91,8 @@ def _normalize_path(raw_path: str) -> str: path = raw_path or "/" path = _normalize_pct_encoding(path) - # RFC treats //a and /a as different resources, but attackers insert - # extra slashes to dodge threat DB pattern matching. + # RFC상 //a와 /a는 다른 리소스지만, 공격자가 슬래시 추가해서 + # 위협 DB 패턴 매칭 우회하는 케이스 방지. path = re.sub(r"/{2,}", "/", path) path = _resolve_dot_segments(path) @@ -101,7 +100,7 @@ def _normalize_path(raw_path: str) -> str: def _resolve_dot_segments(path: str) -> str: - """RFC 3986 §5.2.4""" + """dot segment(. / ..) 해소.""" segments = path.split("/") resolved: list[str] = [] for seg in segments: @@ -118,7 +117,7 @@ def _resolve_dot_segments(path: str) -> str: def _normalize_pct_encoding(value: str) -> str: - """Decode unreserved (%41->A), uppercase reserved hex (%2f->%2F).""" + """비예약 문자 디코딩(%41->A), 예약 문자 hex 대문자화(%2f->%2F).""" if not value: return value diff --git a/tests/demo_normalize.py b/tests/demo_normalize.py index b1038bb..f128277 100644 --- a/tests/demo_normalize.py +++ b/tests/demo_normalize.py @@ -1,6 +1,6 @@ -"""URL normalization demo. +"""URL 정규화 데모. -Usage: +사용법: python -m tests.demo_normalize """ diff --git a/tests/services/normalizer/test_normalize.py b/tests/services/normalizer/test_normalize.py index 213c402..9c886c2 100644 --- a/tests/services/normalizer/test_normalize.py +++ b/tests/services/normalizer/test_normalize.py @@ -1,4 +1,4 @@ -"""normalize_url unit tests.""" +"""normalize_url 단위 테스트.""" import pytest from app.core.exceptions import NormalizationError From 0236a54944ffb358169d0c3c720551c62af1f20f Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:24:23 +0900 Subject: [PATCH 017/173] =?UTF-8?q?[Feat]=20URL=20=EC=96=B8=EC=B2=B4?= =?UTF-8?q?=EC=9D=B4=EB=8B=9D=20=EA=B2=B0=EA=B3=BC=20=EC=8A=A4=ED=82=A4?= =?UTF-8?q?=EB=A7=88=20=EC=B6=94=EA=B0=80=20(HopRecord,=20UnchainResult)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 0da747c..41585c3 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -6,3 +6,27 @@ class NormalizeResult(BaseModel): original_url: str = Field(description="Original URL after trimming") normalized_url: str = Field(description="Canonicalized URL") + + +class HopRecord(BaseModel): + """리다이렉트 체인의 개별 hop.""" + + url: str = Field(description="이 hop에서 요청한 URL") + status_code: int = Field(description="HTTP 응답 상태 코드") + location: str | None = Field(default=None, description="Location 헤더 값(있을 경우)") + method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") + + +class UnchainResult(BaseModel): + """Stage 2 — URL 언체이닝(리다이렉트 추적) 결과.""" + + input_url: str = Field(description="언체이닝 시작 URL") + final_url: str = Field(description="최종 도달 URL") + hops: list[HopRecord] = Field(default_factory=list, description="리다이렉트 hop 기록") + hop_count: int = Field(default=0, description="총 hop 수") + timed_out: bool = Field(default=False, description="타임아웃 발생 여부") + error: str | None = Field(default=None, description="체인 중단 사유(에러 발생 시)") + signals: list[str] = Field( + default_factory=list, + description="탐지된 의심 신호 목록", + ) From 945e19a9d50ab05be5725e206567d552515fb558 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:24:45 +0900 Subject: [PATCH 018/173] =?UTF-8?q?[Feat]=20=EC=96=B8=EC=B2=B4=EC=9D=B4?= =?UTF-8?q?=EB=8B=9D=20=EA=B4=80=EB=A0=A8=20=EC=84=A4=EC=A0=95=EA=B0=92=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80=20(max=5Fhops,=20timeout,=20user=5Fagent)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/core/config.py b/app/core/config.py index d804051..620681f 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -77,6 +77,16 @@ def alembic_database_url(self) -> str: # 정규화 normalizer_max_url_length: int = 1024 + # 언체이닝 + unchain_max_hops: int = 20 + unchain_timeout_seconds: float = 10.0 + unchain_per_hop_timeout_seconds: float = 5.0 + unchain_user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ) + # 점수 산정 score_weight_gsb: int = 50 score_weight_urlhaus: int = 50 From 0724f3f81e4fa2ec9ca1bc5346093ce849fa98d2 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:25:40 +0900 Subject: [PATCH 019/173] =?UTF-8?q?[Feat]=20URL=20=EC=96=B8=EC=B2=B4?= =?UTF-8?q?=EC=9D=B4=EB=8B=9D=20service=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/__init__.py | 3 + app/services/unchainer/unchain.py | 164 +++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 app/services/unchainer/__init__.py create mode 100644 app/services/unchainer/unchain.py diff --git a/app/services/unchainer/__init__.py b/app/services/unchainer/__init__.py new file mode 100644 index 0000000..7d06557 --- /dev/null +++ b/app/services/unchainer/__init__.py @@ -0,0 +1,3 @@ +from app.services.unchainer.unchain import unchain_url + +__all__ = ["unchain_url"] diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py new file mode 100644 index 0000000..8e2b228 --- /dev/null +++ b/app/services/unchainer/unchain.py @@ -0,0 +1,164 @@ +"""URL 언체이닝 — 파이프라인 2단계. + +정규화된 URL의 리다이렉트 체인을 끝까지 추적해서 +최종 목적지 URL과 경로상 의심 신호를 수집함. +""" + +from __future__ import annotations + +import ssl +from socket import gaierror +from urllib.parse import urljoin, urlparse + +import httpx + +from app.core.config import settings +from app.schemas.analysis import HopRecord, UnchainResult + +# 리다이렉트로 간주할 상태 코드 +_REDIRECT_CODES: frozenset[int] = frozenset({301, 302, 303, 307, 308}) + + +async def unchain_url(url: str) -> UnchainResult: + """리다이렉트 체인을 추적하고 최종 URL·hop 기록·의심 신호를 반환.""" + hops: list[HopRecord] = [] + signals: list[str] = [] + visited: set[str] = set() + current_url = url + error: str | None = None + timed_out = False + + headers = { + "User-Agent": settings.unchain_user_agent, + "Accept": "*/*", + "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7", + } + + async with httpx.AsyncClient( + timeout=httpx.Timeout( + settings.unchain_timeout_seconds, + connect=settings.unchain_per_hop_timeout_seconds, + ), + follow_redirects=False, + verify=True, + # 쿠키·인증 비전송 + cookies=None, + ) as client: + for _ in range(settings.unchain_max_hops): + # 무한 루프 감지 + if current_url in visited: + signals.append("redirect_loop") + break + visited.add(current_url) + + hop, next_url, hop_error = await _follow_one_hop( + client, current_url, headers, signals, + ) + + if hop is not None: + hops.append(hop) + + if hop_error is not None: + error = hop_error + if hop_error == "timeout": + timed_out = True + break + + # 리다이렉트가 아니면 체인 종료 + if next_url is None: + break + + current_url = next_url + else: + # max_hops 도달 + signals.append("max_hops_reached") + + return UnchainResult( + input_url=url, + final_url=current_url, + hops=hops, + hop_count=len(hops), + timed_out=timed_out, + error=error, + signals=signals, + ) + + +async def _follow_one_hop( + client: httpx.AsyncClient, + url: str, + headers: dict[str, str], + signals: list[str], +) -> tuple[HopRecord | None, str | None, str | None]: + """단일 hop 요청. (hop_record, next_url_or_None, error_or_None) 반환. + + HEAD 우선 → 실패 시 GET 폴백. + """ + for method in ("HEAD", "GET"): + try: + resp = await client.request(method, url, headers=headers) + except httpx.TimeoutException: + return None, None, "timeout" + except gaierror: + return None, None, "dns_failure" + except ssl.SSLError as e: + return None, None, f"tls_error: {e}" + except httpx.ConnectError as e: + # DNS 실패가 ConnectError 안에 래핑되는 경우도 처리 + cause = str(e).lower() + if "name or service not known" in cause or "getaddrinfo" in cause: + return None, None, "dns_failure" + return None, None, f"connection_refused: {e}" + except httpx.HTTPError as e: + return None, None, f"http_error: {e}" + + # HEAD에서 405 등 클라이언트 에러면 GET 폴백 + if method == "HEAD" and resp.status_code in {405, 403, 400}: + continue + + # 5xx 서버 에러 → 체인 중단 + if resp.status_code >= 500: + hop = HopRecord( + url=url, status_code=resp.status_code, method=method, + ) + return hop, None, f"server_error_{resp.status_code}" + + # 리다이렉트 처리 + if resp.status_code in _REDIRECT_CODES: + location = resp.headers.get("location") + if not location: + hop = HopRecord( + url=url, status_code=resp.status_code, method=method, + ) + return hop, None, "missing_location_header" + + # 상대 경로 Location 해석 + next_url = urljoin(url, location) + + hop = HopRecord( + url=url, + status_code=resp.status_code, + location=next_url, + method=method, + ) + + # 스킴 다운그레이드 감지 (https → http) + if urlparse(url).scheme == "https" and urlparse(next_url).scheme == "http": + signals.append("scheme_downgrade") + + # 크로스 오리진 호스트 변화 + prev_host = urlparse(url).hostname + next_host = urlparse(next_url).hostname + if prev_host != next_host: + signals.append(f"cross_origin:{prev_host}->{next_host}") + + return hop, next_url, None + + # 리다이렉트 아닌 정상 응답 → 체인 종료 + hop = HopRecord( + url=url, status_code=resp.status_code, method=method, + ) + return hop, None, None + + # 여기 도달 불가하지만 타입 안전 + return None, None, "unexpected_fallthrough" # pragma: no cover From 60965cc13325937f58d18556395ae8a027bf909c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:26:15 +0900 Subject: [PATCH 020/173] =?UTF-8?q?[Test]=20URL=20=EC=96=B8=EC=B2=B4?= =?UTF-8?q?=EC=9D=B4=EB=8B=9D=20=EB=8B=A8=EC=9C=84=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8=2017=EA=B1=B4=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/services/unchainer/__init__.py | 0 tests/services/unchainer/test_unchain.py | 343 +++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 tests/services/unchainer/__init__.py create mode 100644 tests/services/unchainer/test_unchain.py diff --git a/tests/services/unchainer/__init__.py b/tests/services/unchainer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py new file mode 100644 index 0000000..5bb1bd5 --- /dev/null +++ b/tests/services/unchainer/test_unchain.py @@ -0,0 +1,343 @@ +"""unchain_url 단위 테스트. + +httpx.AsyncClient를 모킹해서 네트워크 요청 없이 리다이렉트 체인 로직을 검증. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +from app.services.unchainer.unchain import unchain_url + + +def _make_response( + status_code: int = 200, + headers: dict[str, str] | None = None, +) -> httpx.Response: + """테스트용 httpx.Response 생성.""" + return httpx.Response( + status_code=status_code, + headers=headers or {}, + request=httpx.Request("HEAD", "https://example.com"), + ) + + +class TestBasicChain: + """리다이렉트 없는 경우 / 단일·다중 hop 체인.""" + + async def test_no_redirect(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock(return_value=_make_response(200)) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://example.com/") + + assert result.final_url == "https://example.com/" + assert result.hop_count == 1 + assert result.hops[0].status_code == 200 + assert result.error is None + assert not result.signals + + async def test_single_redirect(self) -> None: + responses = [ + _make_response(301, {"location": "https://example.com/new"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://example.com/old") + + assert result.final_url == "https://example.com/new" + assert result.hop_count == 2 + assert result.hops[0].status_code == 301 + assert result.hops[0].location == "https://example.com/new" + assert result.hops[1].status_code == 200 + + async def test_multi_hop_chain(self) -> None: + responses = [ + _make_response(302, {"location": "https://a.com/2"}), + _make_response(302, {"location": "https://a.com/3"}), + _make_response(302, {"location": "https://a.com/final"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://a.com/1") + + assert result.final_url == "https://a.com/final" + assert result.hop_count == 4 + + +class TestRedirectLoop: + """무한 루프 감지.""" + + async def test_loop_detected(self) -> None: + """A → B → A 루프 감지 시 redirect_loop 신호.""" + responses = [ + _make_response(302, {"location": "https://b.com/"}), + _make_response(302, {"location": "https://a.com/"}), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://a.com/") + + assert "redirect_loop" in result.signals + assert result.hop_count == 2 + + +class TestMaxHops: + """hop 수 제한.""" + + async def test_max_hops_reached(self) -> None: + def _redirect(request: httpx.Request) -> httpx.Response: + return _make_response(302, {"location": "https://example.com/next"}) + + # 무한 리다이렉트지만 URL이 매번 같으므로 루프 감지가 먼저 발동됨. + # URL을 매 hop마다 다르게 생성해야 max_hops 테스트 가능. + hop_count = 0 + + def _side_effect(*args: object, **kwargs: object) -> httpx.Response: + nonlocal hop_count + hop_count += 1 + return _make_response( + 302, {"location": f"https://example.com/{hop_count}"}, + ) + + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=_side_effect) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with ( + patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client), + patch("app.services.unchainer.unchain.settings") as mock_settings, + ): + mock_settings.unchain_max_hops = 5 + mock_settings.unchain_timeout_seconds = 10.0 + mock_settings.unchain_per_hop_timeout_seconds = 5.0 + mock_settings.unchain_user_agent = "test-agent" + result = await unchain_url("https://example.com/start") + + assert "max_hops_reached" in result.signals + assert result.hop_count == 5 + + +class TestSchemeDowngrade: + """https → http 스킴 다운그레이드 감지.""" + + async def test_scheme_downgrade_signal(self) -> None: + responses = [ + _make_response(302, {"location": "http://example.com/insecure"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://example.com/") + + assert "scheme_downgrade" in result.signals + + async def test_no_signal_on_http_to_https(self) -> None: + responses = [ + _make_response(302, {"location": "https://example.com/secure"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("http://example.com/") + + assert "scheme_downgrade" not in result.signals + + +class TestCrossOrigin: + """크로스 오리진 호스트 변화 감지.""" + + async def test_cross_origin_signal(self) -> None: + responses = [ + _make_response(302, {"location": "https://evil.com/phish"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://safe.com/link") + + assert any(s.startswith("cross_origin:") for s in result.signals) + assert "cross_origin:safe.com->evil.com" in result.signals + + +class TestRelativeLocation: + """상대 경로 Location 해석.""" + + async def test_relative_path_resolved(self) -> None: + responses = [ + _make_response(302, {"location": "/new-path"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://example.com/old-path") + + assert result.final_url == "https://example.com/new-path" + assert result.hops[0].location == "https://example.com/new-path" + + +class TestHeadFallback: + """HEAD 실패 시 GET 폴백.""" + + async def test_head_405_falls_back_to_get(self) -> None: + responses = [ + _make_response(405), # HEAD 실패 + _make_response(200), # GET 성공 + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + +class TestErrorHandling: + """네트워크·서버 에러 처리.""" + + async def test_timeout_error(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=httpx.ReadTimeout("timed out")) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://slow.example.com/") + + assert result.timed_out is True + assert result.error == "timeout" + + async def test_dns_failure_via_connect_error(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock( + side_effect=httpx.ConnectError( + "[Errno -2] Name or service not known", + ), + ) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://nonexistent.invalid/") + + assert result.error == "dns_failure" + + async def test_connection_refused(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock( + side_effect=httpx.ConnectError("Connection refused"), + ) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://refused.example.com/") + + assert result.error is not None + assert "connection_refused" in result.error + + async def test_server_error_5xx(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock(return_value=_make_response(503)) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://down.example.com/") + + assert result.error == "server_error_503" + assert result.hop_count == 1 + assert result.hops[0].status_code == 503 + + async def test_missing_location_header(self) -> None: + mock_client = AsyncMock() + mock_client.request = AsyncMock(return_value=_make_response(302)) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://broken.example.com/") + + assert result.error == "missing_location_header" + + +class TestHopRecording: + """hop 기록 정확성.""" + + async def test_hop_records_method_and_url(self) -> None: + responses = [ + _make_response(302, {"location": "https://b.com/"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + result = await unchain_url("https://a.com/") + + assert result.hops[0].url == "https://a.com/" + assert result.hops[0].method == "HEAD" + assert result.hops[1].url == "https://b.com/" + + async def test_all_redirect_status_codes(self) -> None: + """301, 302, 303, 307, 308 모두 리다이렉트로 처리.""" + for code in (301, 302, 303, 307, 308): + responses = [ + _make_response(code, {"location": "https://example.com/dest"}), + _make_response(200), + ] + mock_client = AsyncMock() + mock_client.request = AsyncMock(side_effect=responses) + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + + with patch( + "app.services.unchainer.unchain.httpx.AsyncClient", + return_value=mock_client, + ): + result = await unchain_url("https://example.com/src") + + assert result.hops[0].status_code == code, f"Failed for status {code}" + assert result.final_url == "https://example.com/dest" From e70dff8b51f8fe58592f9b2704080d6a6e88efce Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:27:05 +0900 Subject: [PATCH 021/173] =?UTF-8?q?[Chore]=20=EC=96=B8=EC=B2=B4=EC=9D=B4?= =?UTF-8?q?=EB=8B=9D=20=EB=8D=B0=EB=AA=A8=20=EC=8A=A4=ED=81=AC=EB=A6=BD?= =?UTF-8?q?=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo_unchain.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/demo_unchain.py diff --git a/tests/demo_unchain.py b/tests/demo_unchain.py new file mode 100644 index 0000000..b9ae177 --- /dev/null +++ b/tests/demo_unchain.py @@ -0,0 +1,60 @@ +"""URL 언체이닝 데모. + +사용법: + python -m tests.demo_unchain +""" + +from __future__ import annotations + +import asyncio + +from app.services.unchainer import unchain_url + +DIVIDER = "-" * 72 + +# 실제 리다이렉트가 일어나는 공개 URL들 (데모 시점에 동작 보장 불가) +CASES: list[tuple[str, str]] = [ + ("리다이렉트 없음", "https://www.google.com/"), + ("HTTP → HTTPS 업그레이드", "http://google.com/"), + ("단축 URL (bit.ly 예시)", "https://bit.ly/3kF8Yv2"), + ("t.co 단축 URL", "https://t.co/example"), + ("HTTP 301 리다이렉트", "http://github.com/"), + ("상대 경로 리다이렉트", "https://httpbin.org/relative-redirect/2"), + ("절대 경로 리다이렉트", "https://httpbin.org/absolute-redirect/3"), + ("존재하지 않는 도메인 (DNS 실패)", "https://this-domain-does-not-exist-12345.com/"), +] + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- URL Unchaining Demo") + print("=" * 72) + + for i, (desc, url) in enumerate(CASES, 1): + print(f"\n [{i:02d}] {desc}") + print(f" input : {url}") + + result = await unchain_url(url) + + print(f" final : {result.final_url}") + print(f" hops : {result.hop_count}") + + for j, hop in enumerate(result.hops): + loc = f" -> {hop.location}" if hop.location else "" + print(f" [{j}] {hop.method} {hop.status_code}{loc}") + + if result.signals: + print(f" signals: {result.signals}") + if result.error: + print(f" error : {result.error}") + if result.timed_out: + print(" ⚠ TIMED OUT") + + print(DIVIDER) + + print() + + +if __name__ == "__main__": + asyncio.run(main()) From 12b952c1d6ed19ca76765564a77875c466310e08 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:29:53 +0900 Subject: [PATCH 022/173] =?UTF-8?q?[Chore]=20demo=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A6=BD=ED=8A=B8=EB=A5=BC=20tests/demo=20=EB=94=94=EB=A0=89?= =?UTF-8?q?=ED=86=A0=EB=A6=AC=EB=A1=9C=20=EC=9D=B4=EB=8F=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/__init__.py | 0 tests/{ => demo}/demo_normalize.py | 2 +- tests/{ => demo}/demo_unchain.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/demo/__init__.py rename tests/{ => demo}/demo_normalize.py (98%) rename tests/{ => demo}/demo_unchain.py (97%) diff --git a/tests/demo/__init__.py b/tests/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/demo_normalize.py b/tests/demo/demo_normalize.py similarity index 98% rename from tests/demo_normalize.py rename to tests/demo/demo_normalize.py index f128277..4aa1c56 100644 --- a/tests/demo_normalize.py +++ b/tests/demo/demo_normalize.py @@ -1,7 +1,7 @@ """URL 정규화 데모. 사용법: - python -m tests.demo_normalize + python -m tests.demo.demo_normalize """ from __future__ import annotations diff --git a/tests/demo_unchain.py b/tests/demo/demo_unchain.py similarity index 97% rename from tests/demo_unchain.py rename to tests/demo/demo_unchain.py index b9ae177..d527838 100644 --- a/tests/demo_unchain.py +++ b/tests/demo/demo_unchain.py @@ -1,7 +1,7 @@ """URL 언체이닝 데모. 사용법: - python -m tests.demo_unchain + python -m tests.demo.demo_unchain """ from __future__ import annotations From fc5bf84131495e31f6aac839166bcd4bfa085201 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:54:45 +0900 Subject: [PATCH 023/173] =?UTF-8?q?[Refactor]=20=EC=96=B8=EC=B2=B4?= =?UTF-8?q?=EC=9D=B4=EB=8B=9D=20timeout=20=EC=84=A4=EC=A0=95=20=EC=A1=B0?= =?UTF-8?q?=EC=A0=95=20=EB=B0=8F=20=EC=84=A4=EC=A0=95=EB=AA=85=20=EB=AA=85?= =?UTF-8?q?=ED=99=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/core/config.py b/app/core/config.py index 620681f..bb3a2f2 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -79,8 +79,9 @@ def alembic_database_url(self) -> str: # 언체이닝 unchain_max_hops: int = 20 - unchain_timeout_seconds: float = 10.0 - unchain_per_hop_timeout_seconds: float = 5.0 + unchain_timeout_seconds: float = 5.0 + unchain_connect_timeout_seconds: float = 3.0 + unchain_chain_timeout_seconds: float = 20.0 unchain_user_agent: str = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " From 8a5c5559925d9ea0450645db47ee06b4f696049b Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:55:09 +0900 Subject: [PATCH 024/173] =?UTF-8?q?[Feat]=20HopRecord=EC=97=90=20raw=5Floc?= =?UTF-8?q?ation=20=ED=95=84=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 41585c3..a8e6826 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -13,7 +13,8 @@ class HopRecord(BaseModel): url: str = Field(description="이 hop에서 요청한 URL") status_code: int = Field(description="HTTP 응답 상태 코드") - location: str | None = Field(default=None, description="Location 헤더 값(있을 경우)") + raw_location: str | None = Field(default=None, description="Location 헤더 원본 값(상대경로 포함)") + location: str | None = Field(default=None, description="절대경로로 해석된 Location 값") method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") From 2bff33308dc7eabe23dd2efb6bade6c496ece28a Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:57:51 +0900 Subject: [PATCH 025/173] =?UTF-8?q?[Refactor]=20dead=20code=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0,=20HEAD=20=EC=97=90=EB=9F=AC=20=EC=8B=9C=20GET=20?= =?UTF-8?q?=ED=8F=B4=EB=B0=B1,=20=EC=B2=B4=EC=9D=B8=20=EC=B4=9D=20timeout,?= =?UTF-8?q?=20=EB=B9=84=ED=97=88=EC=9A=A9=20=EC=8A=A4=ED=82=B4=20=EB=B0=A9?= =?UTF-8?q?=EC=96=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/unchain.py | 72 +++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index 8e2b228..e6f65b7 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -6,8 +6,7 @@ from __future__ import annotations -import ssl -from socket import gaierror +import asyncio from urllib.parse import urljoin, urlparse import httpx @@ -18,9 +17,31 @@ # 리다이렉트로 간주할 상태 코드 _REDIRECT_CODES: frozenset[int] = frozenset({301, 302, 303, 307, 308}) +# 리다이렉트 대상으로 허용하는 스킴 (javascript:, data: 등 차단) +_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"}) + async def unchain_url(url: str) -> UnchainResult: """리다이렉트 체인을 추적하고 최종 URL·hop 기록·의심 신호를 반환.""" + try: + return await asyncio.wait_for( + _unchain_url_inner(url), + timeout=settings.unchain_chain_timeout_seconds, + ) + except asyncio.TimeoutError: + return UnchainResult( + input_url=url, + final_url=url, + hops=[], + hop_count=0, + timed_out=True, + error="chain_timeout", + signals=[], + ) + + +async def _unchain_url_inner(url: str) -> UnchainResult: + """실제 체인 추적 로직. unchain_url에서 총 timeout으로 감싸서 호출.""" hops: list[HopRecord] = [] signals: list[str] = [] visited: set[str] = set() @@ -37,11 +58,10 @@ async def unchain_url(url: str) -> UnchainResult: async with httpx.AsyncClient( timeout=httpx.Timeout( settings.unchain_timeout_seconds, - connect=settings.unchain_per_hop_timeout_seconds, + connect=settings.unchain_connect_timeout_seconds, ), follow_redirects=False, verify=True, - # 쿠키·인증 비전송 cookies=None, ) as client: for _ in range(settings.unchain_max_hops): @@ -92,24 +112,25 @@ async def _follow_one_hop( ) -> tuple[HopRecord | None, str | None, str | None]: """단일 hop 요청. (hop_record, next_url_or_None, error_or_None) 반환. - HEAD 우선 → 실패 시 GET 폴백. + HEAD 우선 → 실패 시(상태 코드·네트워크 에러 모두) GET 폴백. """ for method in ("HEAD", "GET"): try: resp = await client.request(method, url, headers=headers) except httpx.TimeoutException: + if method == "HEAD": + continue return None, None, "timeout" - except gaierror: - return None, None, "dns_failure" - except ssl.SSLError as e: - return None, None, f"tls_error: {e}" except httpx.ConnectError as e: - # DNS 실패가 ConnectError 안에 래핑되는 경우도 처리 + if method == "HEAD": + continue cause = str(e).lower() if "name or service not known" in cause or "getaddrinfo" in cause: return None, None, "dns_failure" return None, None, f"connection_refused: {e}" except httpx.HTTPError as e: + if method == "HEAD": + continue return None, None, f"http_error: {e}" # HEAD에서 405 등 클라이언트 에러면 GET 폴백 @@ -125,32 +146,45 @@ async def _follow_one_hop( # 리다이렉트 처리 if resp.status_code in _REDIRECT_CODES: - location = resp.headers.get("location") - if not location: + raw_location = resp.headers.get("location") + if not raw_location: hop = HopRecord( url=url, status_code=resp.status_code, method=method, ) return hop, None, "missing_location_header" - # 상대 경로 Location 해석 - next_url = urljoin(url, location) + next_url = urljoin(url, raw_location) + parsed_next = urlparse(next_url) + + # 허용되지 않는 스킴 방어 (javascript:, data: 등) + if parsed_next.scheme not in _ALLOWED_SCHEMES: + hop = HopRecord( + url=url, + status_code=resp.status_code, + raw_location=raw_location, + location=next_url, + method=method, + ) + signals.append(f"unsafe_scheme:{parsed_next.scheme}") + return hop, None, f"unsafe_redirect_scheme:{parsed_next.scheme}" + + parsed_url = urlparse(url) hop = HopRecord( url=url, status_code=resp.status_code, + raw_location=raw_location, location=next_url, method=method, ) # 스킴 다운그레이드 감지 (https → http) - if urlparse(url).scheme == "https" and urlparse(next_url).scheme == "http": + if parsed_url.scheme == "https" and parsed_next.scheme == "http": signals.append("scheme_downgrade") # 크로스 오리진 호스트 변화 - prev_host = urlparse(url).hostname - next_host = urlparse(next_url).hostname - if prev_host != next_host: - signals.append(f"cross_origin:{prev_host}->{next_host}") + if parsed_url.hostname != parsed_next.hostname: + signals.append(f"cross_origin:{parsed_url.hostname}->{parsed_next.hostname}") return hop, next_url, None From 2a5c70bb2dc26eb88aa78f4f9be3de96f8fee923 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:58:18 +0900 Subject: [PATCH 026/173] =?UTF-8?q?[Test]=20=EC=96=B8=EC=B2=B4=EC=9D=B4?= =?UTF-8?q?=EB=8B=9D=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EA=B0=9C=EC=84=A0=20?= =?UTF-8?q?=E2=80=94=20mock=20=ED=97=AC=ED=8D=BC=20=EC=B6=94=EC=B6=9C,=20?= =?UTF-8?q?=EC=8B=A0=EA=B7=9C=20=ED=85=8C=EC=8A=A4=ED=8A=B8=206=EA=B1=B4?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80=20(=EC=B4=9D=2023=EA=B1=B4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/services/unchainer/test_unchain.py | 292 +++++++++++++++-------- 1 file changed, 193 insertions(+), 99 deletions(-) diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py index 5bb1bd5..16436e6 100644 --- a/tests/services/unchainer/test_unchain.py +++ b/tests/services/unchainer/test_unchain.py @@ -5,9 +5,12 @@ from __future__ import annotations +import asyncio from unittest.mock import AsyncMock, patch import httpx +import pytest + from app.services.unchainer.unchain import unchain_url @@ -23,16 +26,35 @@ def _make_response( ) +def _mock_client( + responses: list[httpx.Response] | None = None, + *, + side_effect=None, +) -> AsyncMock: + """httpx.AsyncClient mock 생성. 반복되는 보일러플레이트 제거용.""" + mock = AsyncMock() + if side_effect is not None: + mock.request = AsyncMock(side_effect=side_effect) + elif responses is not None: + mock.request = AsyncMock(side_effect=responses) + else: + mock.request = AsyncMock(return_value=_make_response(200)) + mock.__aenter__ = AsyncMock(return_value=mock) + mock.__aexit__ = AsyncMock(return_value=False) + return mock + + +_PATCH_TARGET = "app.services.unchainer.unchain.httpx.AsyncClient" + + class TestBasicChain: """리다이렉트 없는 경우 / 단일·다중 hop 체인.""" + @pytest.mark.asyncio async def test_no_redirect(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock(return_value=_make_response(200)) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client([_make_response(200)]) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/") assert result.final_url == "https://example.com/" @@ -41,17 +63,15 @@ async def test_no_redirect(self) -> None: assert result.error is None assert not result.signals + @pytest.mark.asyncio async def test_single_redirect(self) -> None: responses = [ _make_response(301, {"location": "https://example.com/new"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/old") assert result.final_url == "https://example.com/new" @@ -60,6 +80,7 @@ async def test_single_redirect(self) -> None: assert result.hops[0].location == "https://example.com/new" assert result.hops[1].status_code == 200 + @pytest.mark.asyncio async def test_multi_hop_chain(self) -> None: responses = [ _make_response(302, {"location": "https://a.com/2"}), @@ -67,12 +88,9 @@ async def test_multi_hop_chain(self) -> None: _make_response(302, {"location": "https://a.com/final"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://a.com/1") assert result.final_url == "https://a.com/final" @@ -82,18 +100,16 @@ async def test_multi_hop_chain(self) -> None: class TestRedirectLoop: """무한 루프 감지.""" + @pytest.mark.asyncio async def test_loop_detected(self) -> None: """A → B → A 루프 감지 시 redirect_loop 신호.""" responses = [ _make_response(302, {"location": "https://b.com/"}), _make_response(302, {"location": "https://a.com/"}), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://a.com/") assert "redirect_loop" in result.signals @@ -103,12 +119,8 @@ async def test_loop_detected(self) -> None: class TestMaxHops: """hop 수 제한.""" + @pytest.mark.asyncio async def test_max_hops_reached(self) -> None: - def _redirect(request: httpx.Request) -> httpx.Response: - return _make_response(302, {"location": "https://example.com/next"}) - - # 무한 리다이렉트지만 URL이 매번 같으므로 루프 감지가 먼저 발동됨. - # URL을 매 hop마다 다르게 생성해야 max_hops 테스트 가능. hop_count = 0 def _side_effect(*args: object, **kwargs: object) -> httpx.Response: @@ -118,18 +130,16 @@ def _side_effect(*args: object, **kwargs: object) -> httpx.Response: 302, {"location": f"https://example.com/{hop_count}"}, ) - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=_side_effect) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(side_effect=_side_effect) with ( - patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client), + patch(_PATCH_TARGET, return_value=client), patch("app.services.unchainer.unchain.settings") as mock_settings, ): mock_settings.unchain_max_hops = 5 - mock_settings.unchain_timeout_seconds = 10.0 - mock_settings.unchain_per_hop_timeout_seconds = 5.0 + mock_settings.unchain_timeout_seconds = 5.0 + mock_settings.unchain_connect_timeout_seconds = 3.0 + mock_settings.unchain_chain_timeout_seconds = 20.0 mock_settings.unchain_user_agent = "test-agent" result = await unchain_url("https://example.com/start") @@ -140,32 +150,28 @@ def _side_effect(*args: object, **kwargs: object) -> httpx.Response: class TestSchemeDowngrade: """https → http 스킴 다운그레이드 감지.""" + @pytest.mark.asyncio async def test_scheme_downgrade_signal(self) -> None: responses = [ _make_response(302, {"location": "http://example.com/insecure"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/") assert "scheme_downgrade" in result.signals + @pytest.mark.asyncio async def test_no_signal_on_http_to_https(self) -> None: responses = [ _make_response(302, {"location": "https://example.com/secure"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("http://example.com/") assert "scheme_downgrade" not in result.signals @@ -174,17 +180,15 @@ async def test_no_signal_on_http_to_https(self) -> None: class TestCrossOrigin: """크로스 오리진 호스트 변화 감지.""" + @pytest.mark.asyncio async def test_cross_origin_signal(self) -> None: responses = [ _make_response(302, {"location": "https://evil.com/phish"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://safe.com/link") assert any(s.startswith("cross_origin:") for s in result.signals) @@ -194,37 +198,87 @@ async def test_cross_origin_signal(self) -> None: class TestRelativeLocation: """상대 경로 Location 해석.""" + @pytest.mark.asyncio async def test_relative_path_resolved(self) -> None: responses = [ _make_response(302, {"location": "/new-path"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/old-path") assert result.final_url == "https://example.com/new-path" assert result.hops[0].location == "https://example.com/new-path" + @pytest.mark.asyncio + async def test_raw_location_preserved(self) -> None: + """상대 경로 원본 값이 raw_location에 보존되는지 확인.""" + responses = [ + _make_response(302, {"location": "/new-path"}), + _make_response(200), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/old-path") + + assert result.hops[0].raw_location == "/new-path" + assert result.hops[0].location == "https://example.com/new-path" + class TestHeadFallback: """HEAD 실패 시 GET 폴백.""" + @pytest.mark.asyncio async def test_head_405_falls_back_to_get(self) -> None: responses = [ _make_response(405), # HEAD 실패 _make_response(200), # GET 성공 ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + @pytest.mark.asyncio + async def test_head_timeout_falls_back_to_get(self) -> None: + """HEAD에서 타임아웃 → GET으로 폴백해서 성공.""" + call_count = 0 + + def _side_effect(method: str, *args: object, **kwargs: object) -> httpx.Response: + nonlocal call_count + call_count += 1 + if method == "HEAD": + raise httpx.ReadTimeout("timed out") + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.hop_count == 1 + assert result.hops[0].method == "GET" + assert result.error is None + + @pytest.mark.asyncio + async def test_head_connect_error_falls_back_to_get(self) -> None: + """HEAD에서 ConnectError → GET으로 폴백해서 성공.""" + + def _side_effect(method: str, *args: object, **kwargs: object) -> httpx.Response: + if method == "HEAD": + raise httpx.ConnectError("Connection refused") + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/") assert result.hop_count == 1 @@ -235,92 +289,138 @@ async def test_head_405_falls_back_to_get(self) -> None: class TestErrorHandling: """네트워크·서버 에러 처리.""" + @pytest.mark.asyncio async def test_timeout_error(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=httpx.ReadTimeout("timed out")) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + """HEAD·GET 모두 타임아웃이면 최종 에러.""" + client = _mock_client(side_effect=httpx.ReadTimeout("timed out")) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://slow.example.com/") assert result.timed_out is True assert result.error == "timeout" + @pytest.mark.asyncio async def test_dns_failure_via_connect_error(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock( - side_effect=httpx.ConnectError( - "[Errno -2] Name or service not known", - ), + client = _mock_client( + side_effect=httpx.ConnectError("[Errno -2] Name or service not known"), ) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://nonexistent.invalid/") assert result.error == "dns_failure" + @pytest.mark.asyncio async def test_connection_refused(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock( + client = _mock_client( side_effect=httpx.ConnectError("Connection refused"), ) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://refused.example.com/") assert result.error is not None assert "connection_refused" in result.error + @pytest.mark.asyncio async def test_server_error_5xx(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock(return_value=_make_response(503)) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client([_make_response(503)]) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://down.example.com/") assert result.error == "server_error_503" assert result.hop_count == 1 assert result.hops[0].status_code == 503 + @pytest.mark.asyncio async def test_missing_location_header(self) -> None: - mock_client = AsyncMock() - mock_client.request = AsyncMock(return_value=_make_response(302)) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client([_make_response(302)]) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://broken.example.com/") assert result.error == "missing_location_header" +class TestChainTimeout: + """전체 체인 총 timeout.""" + + @pytest.mark.asyncio + async def test_chain_timeout(self) -> None: + """hop별로는 안 터지지만 전체 체인이 총 timeout을 초과하면 중단.""" + + async def _slow_request(*args: object, **kwargs: object) -> httpx.Response: + await asyncio.sleep(10) + return _make_response(302, {"location": "https://example.com/next"}) + + client = _mock_client(side_effect=_slow_request) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch("app.services.unchainer.unchain.settings") as mock_settings, + ): + mock_settings.unchain_max_hops = 20 + mock_settings.unchain_timeout_seconds = 5.0 + mock_settings.unchain_connect_timeout_seconds = 3.0 + mock_settings.unchain_chain_timeout_seconds = 0.1 + mock_settings.unchain_user_agent = "test-agent" + result = await unchain_url("https://example.com/start") + + assert result.timed_out is True + assert result.error == "chain_timeout" + + +class TestUnsafeScheme: + """javascript:, data: 등 비허용 스킴 리다이렉트 차단.""" + + @pytest.mark.asyncio + async def test_javascript_scheme_blocked(self) -> None: + responses = [ + _make_response(302, {"location": "javascript:alert(1)"}), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.error == "unsafe_redirect_scheme:javascript" + assert "unsafe_scheme:javascript" in result.signals + + @pytest.mark.asyncio + async def test_data_scheme_blocked(self) -> None: + responses = [ + _make_response(302, {"location": "data:text/html,

phish

"}), + ] + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/") + + assert result.error == "unsafe_redirect_scheme:data" + assert "unsafe_scheme:data" in result.signals + + class TestHopRecording: """hop 기록 정확성.""" + @pytest.mark.asyncio async def test_hop_records_method_and_url(self) -> None: responses = [ _make_response(302, {"location": "https://b.com/"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) + client = _mock_client(responses) - with patch("app.services.unchainer.unchain.httpx.AsyncClient", return_value=mock_client): + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://a.com/") assert result.hops[0].url == "https://a.com/" assert result.hops[0].method == "HEAD" assert result.hops[1].url == "https://b.com/" + @pytest.mark.asyncio async def test_all_redirect_status_codes(self) -> None: """301, 302, 303, 307, 308 모두 리다이렉트로 처리.""" for code in (301, 302, 303, 307, 308): @@ -328,15 +428,9 @@ async def test_all_redirect_status_codes(self) -> None: _make_response(code, {"location": "https://example.com/dest"}), _make_response(200), ] - mock_client = AsyncMock() - mock_client.request = AsyncMock(side_effect=responses) - mock_client.__aenter__ = AsyncMock(return_value=mock_client) - mock_client.__aexit__ = AsyncMock(return_value=False) - - with patch( - "app.services.unchainer.unchain.httpx.AsyncClient", - return_value=mock_client, - ): + client = _mock_client(responses) + + with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://example.com/src") assert result.hops[0].status_code == code, f"Failed for status {code}" From c63ef6d3f2edd9b0f70abb3b3e41ec612d107c5e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 11:58:35 +0900 Subject: [PATCH 027/173] =?UTF-8?q?[Docs]=20README=EC=97=90=20=EC=96=B8?= =?UTF-8?q?=EC=B2=B4=EC=9D=B4=EB=8B=9D=20=EB=AA=A8=EB=93=88=20=EB=B0=98?= =?UTF-8?q?=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0218eb7..a44a9ad 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,12 @@ linclean-fastapi/ │ │ └── analysis.py # NormalizeResult 등 파이프라인 단계별 DTO │ │ │ └── services/ # 도메인/비즈니스 로직 (파이프라인 단계별 모듈) -│ └── normalizer/ # 1단계: URL 정규화(Canonicalization) -│ ├── __init__.py # 진입점 (normalize_url re-export) -│ └── normalize.py # 입력 검증, 스킴·호스트 정규화, 포트·프래그먼트 제거, 퍼센트 인코딩 정돈, 경로 정규화, IDN 디코딩 +│ ├── normalizer/ # 1단계: URL 정규화(Canonicalization) +│ │ ├── __init__.py # 진입점 (normalize_url re-export) +│ │ └── normalize.py # 입력 검증, 스킴·호스트 정규화, 포트·프래그먼트 제거, 퍼센트 인코딩 정돈, 경로 정규화, IDN 디코딩 +│ └── unchainer/ # 1-2단계: URL 언체이닝(리다이렉트 추적) +│ ├── __init__.py # 진입점 (unchain_url re-export) +│ └── unchain.py # HEAD+GET 폴백, 체인 총 timeout, 스킴 방어, 의심 신호 수집 │ ├── alembic/ # 외부 피드 캐시 스키마 마이그레이션 │ ├── env.py # SQLite + batch mode 설정 @@ -76,10 +79,14 @@ linclean-fastapi/ │ ├── tests/ # pytest 테스트 │ ├── conftest.py # 공용 픽스처 -│ ├── demo_normalize.py # URL 정규화 데모 스크립트 +│ ├── demo/ # 데모 스크립트 +│ │ ├── demo_normalize.py # URL 정규화 데모 +│ │ └── demo_unchain.py # URL 언체이닝 데모 │ └── services/ -│ └── normalizer/ -│ └── test_normalize.py # URL 정규화 단위 테스트 (38개) +│ ├── normalizer/ +│ │ └── test_normalize.py # URL 정규화 단위 테스트 (38개) +│ └── unchainer/ +│ └── test_unchain.py # URL 언체이닝 단위 테스트 (23개) │ ├── docs/ # 기획서 등 설계 문서 │ └── LinClean_기능_초안.pdf @@ -115,6 +122,12 @@ linclean-fastapi/ - **`normalizer/`** — 1단계. `normalize_url()` 로 URL 을 canonical form 으로 정규화합니다 (스킴·호스트 소문자화, 기본 포트 제거, 퍼센트 인코딩 정돈, 경로 dot-segment 해소, IDN 디코딩, 프래그먼트 제거, 입력 검증). + - **`unchainer/`** — 1-2단계. `unchain_url()` 로 리다이렉트 체인(3xx Location) + 을 끝까지 추적해 최종 URL 을 확정합니다. HEAD 우선 → GET 폴백 전략으로 + 대역폭을 절약하면서 호환성을 확보하고, 네트워크 에러 시에도 GET 으로 + 재시도합니다. 체인 전체에 총 timeout 을 적용해 악의적 서버 방어가 가능하며, + `javascript:` / `data:` 같은 비허용 스킴 리다이렉트를 차단합니다. + 스킴 다운그레이드·크로스 오리진 등의 의심 신호도 수집합니다. - **`middleware/`** — `RequestContextMiddleware` 가 매 요청마다 `X-Request-ID` 를 생성/전파하고 structlog contextvars 에 바인딩합니다. 응답 헤더로도 echo 되며 모든 로그 라인에 자동으로 따라붙습니다. @@ -158,9 +171,13 @@ Spring `/internal/analysis-result` 콜백 POST - **정규화**: 스킴·호스트 소문자화, 기본 포트(`:80` / `:443`) 제거, 프래그먼트(`#...`) 제거, 퍼센트 인코딩 정돈, 추적 파라미터(`utm_*` 등) 정책적 제거, IDN(퓨니코드) → 유니코드 정규화 -- **언체이닝**: `HEAD` 요청만으로 리다이렉트 체인(3xx Location) 을 끝까지 - 따라가 **최종 분석 대상 URL** 을 확정합니다. 무한 루프 방지를 위해 방문한 - URL 을 추적하며, hop 수가 5 이상이면 자체적으로 의심 신호로 가산합니다. +- **언체이닝**: `HEAD` 우선 → `GET` 폴백 전략으로 리다이렉트 체인(3xx Location) + 을 끝까지 따라가 **최종 분석 대상 URL** 을 확정합니다. + - 네트워크 에러(타임아웃·연결 실패 등) 발생 시에도 GET 으로 재시도 + - 체인 전체에 총 timeout(기본 30초) 적용 — 악의적 서버의 지연 공격 방어 + - `javascript:`, `data:` 등 비허용 스킴 리다이렉트 차단 + - 스킴 다운그레이드(HTTPS→HTTP), 크로스 오리진, 무한 루프, max hop 초과 감지 + - 각 hop 의 원본 Location 값(`raw_location`)과 절대경로 해석 결과를 모두 기록 - 이후 2~4 단계는 모두 이 **최종 URL** 을 기준으로 동작합니다. ### 2단계 — 외부 위협 DB 대조 From e155b24605e2d99374c55b2c1420cbb82b0c7222 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 13:28:43 +0900 Subject: [PATCH 028/173] =?UTF-8?q?[Fix]=20SSRF=C2=B7DNS=20Rebinding=C2=B7?= =?UTF-8?q?=EC=9D=91=EB=8B=B5=20=EB=B3=B8=EB=AC=B8=20=EB=A9=94=EB=AA=A8?= =?UTF-8?q?=EB=A6=AC=20=EC=B7=A8=EC=95=BD=EC=A0=90=20=EB=B0=A9=EC=96=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/unchain.py | 52 ++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index e6f65b7..7365cee 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -7,6 +7,8 @@ from __future__ import annotations import asyncio +import ipaddress +import socket from urllib.parse import urljoin, urlparse import httpx @@ -21,6 +23,43 @@ _ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"}) +def _is_private_ip(ip_str: str) -> bool: + """외부 접근 불가(is_global=False)인 IP인지 확인. + + is_reserved는 NAT64(64:ff9b::/96) 같은 공개 도달 가능 대역까지 잡아서 + 오탐이 발생하므로, is_global 기반으로 판단. + """ + try: + addr = ipaddress.ip_address(ip_str) + except ValueError: + return True # 파싱 실패 시 안전하게 차단 + return not addr.is_global + + +async def _check_host_safety(url: str) -> str | None: + """호스트를 DNS 조회 후 내부 IP이면 에러 사유 반환, 안전하면 None.""" + parsed = urlparse(url) + hostname = parsed.hostname + if not hostname: + return "invalid_host" + + port = parsed.port or (443 if parsed.scheme == "https" else 80) + loop = asyncio.get_running_loop() + + try: + addrinfo = await loop.getaddrinfo( + hostname, port, type=socket.SOCK_STREAM, + ) + except socket.gaierror: + return "dns_failure" + + for _, _, _, _, sockaddr in addrinfo: + if _is_private_ip(sockaddr[0]): + return "ssrf_blocked" + + return None + + async def unchain_url(url: str) -> UnchainResult: """리다이렉트 체인을 추적하고 최종 URL·hop 기록·의심 신호를 반환.""" try: @@ -71,6 +110,14 @@ async def _unchain_url_inner(url: str) -> UnchainResult: break visited.add(current_url) + # SSRF·DNS Rebinding 방어: 매 hop마다 목적지 IP 검증 + safety_error = await _check_host_safety(current_url) + if safety_error is not None: + error = safety_error + if safety_error == "ssrf_blocked": + signals.append("ssrf_blocked") + break + hop, next_url, hop_error = await _follow_one_hop( client, current_url, headers, signals, ) @@ -116,7 +163,10 @@ async def _follow_one_hop( """ for method in ("HEAD", "GET"): try: - resp = await client.request(method, url, headers=headers) + # 헤더만 필요하므로 스트리밍으로 열고 본문은 즉시 닫음 + req = client.build_request(method, url, headers=headers) + resp = await client.send(req, stream=True) + await resp.aclose() except httpx.TimeoutException: if method == "HEAD": continue From c69a13a507d9811299dd6b132256378bf9cfad46 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 13:29:09 +0900 Subject: [PATCH 029/173] =?UTF-8?q?[Test]=20SSRF=20=EB=B0=A9=EC=96=B4=20?= =?UTF-8?q?=ED=85=8C=EC=8A=A4=ED=8A=B8=2010=EA=B1=B4=20=EC=B6=94=EA=B0=80,?= =?UTF-8?q?=20mock=20=ED=97=AC=ED=8D=BC=20=EC=8A=A4=ED=8A=B8=EB=A6=AC?= =?UTF-8?q?=EB=B0=8D=20=EB=8C=80=EC=9D=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/services/unchainer/test_unchain.py | 170 ++++++++++++++++++++++- 1 file changed, 165 insertions(+), 5 deletions(-) diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py index 16436e6..f8e29db 100644 --- a/tests/services/unchainer/test_unchain.py +++ b/tests/services/unchainer/test_unchain.py @@ -19,11 +19,13 @@ def _make_response( headers: dict[str, str] | None = None, ) -> httpx.Response: """테스트용 httpx.Response 생성.""" - return httpx.Response( + resp = httpx.Response( status_code=status_code, headers=headers or {}, request=httpx.Request("HEAD", "https://example.com"), ) + resp.aclose = AsyncMock() # 스트리밍 close 호출 안전하게 처리 + return resp def _mock_client( @@ -31,14 +33,33 @@ def _mock_client( *, side_effect=None, ) -> AsyncMock: - """httpx.AsyncClient mock 생성. 반복되는 보일러플레이트 제거용.""" + """httpx.AsyncClient mock 생성. 반복되는 보일러플레이트 제거용. + + build_request + send(stream=True) 방식에 맞춰 mock 구성. + side_effect 함수는 기존과 동일하게 (method, *args, **kwargs) 시그니처. + """ mock = AsyncMock() + + # build_request는 동기 메서드 — 실제 httpx.Request 반환 + mock.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + if side_effect is not None: - mock.request = AsyncMock(side_effect=side_effect) + if isinstance(side_effect, BaseException): + mock.send = AsyncMock(side_effect=side_effect) + else: + _original = side_effect + + async def _send_wrapper(req, **kwargs): + if asyncio.iscoroutinefunction(_original): + return await _original(req.method, str(req.url)) + return _original(req.method, str(req.url)) + + mock.send = AsyncMock(side_effect=_send_wrapper) elif responses is not None: - mock.request = AsyncMock(side_effect=responses) + mock.send = AsyncMock(side_effect=responses) else: - mock.request = AsyncMock(return_value=_make_response(200)) + mock.send = AsyncMock(return_value=_make_response(200)) + mock.__aenter__ = AsyncMock(return_value=mock) mock.__aexit__ = AsyncMock(return_value=False) return mock @@ -47,6 +68,15 @@ def _mock_client( _PATCH_TARGET = "app.services.unchainer.unchain.httpx.AsyncClient" +@pytest.fixture(autouse=True) +def _bypass_host_safety(monkeypatch): + """기본적으로 SSRF 검사를 우회 — SSRF 전용 테스트에서만 직접 패치.""" + monkeypatch.setattr( + "app.services.unchainer.unchain._check_host_safety", + AsyncMock(return_value=None), + ) + + class TestBasicChain: """리다이렉트 없는 경우 / 단일·다중 hop 체인.""" @@ -435,3 +465,133 @@ async def test_all_redirect_status_codes(self) -> None: assert result.hops[0].status_code == code, f"Failed for status {code}" assert result.final_url == "https://example.com/dest" + + +class TestSsrfProtection: + """SSRF·DNS Rebinding 방어.""" + + @pytest.mark.asyncio + async def test_loopback_blocked(self) -> None: + """127.0.0.1 등 루프백 주소 차단.""" + with patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ): + result = await unchain_url("http://127.0.0.1/admin") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + assert result.hop_count == 0 + + @pytest.mark.asyncio + async def test_private_ip_blocked(self) -> None: + """사설 IP 대역(10.x, 172.16.x, 192.168.x) 차단.""" + for url in ( + "http://10.0.0.1/", + "http://172.16.0.1/", + "http://192.168.1.1/", + ): + with patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ): + result = await unchain_url(url) + + assert result.error == "ssrf_blocked", f"Failed for {url}" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_metadata_endpoint_blocked(self) -> None: + """클라우드 메타데이터 엔드포인트(169.254.169.254) 차단.""" + with patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value="ssrf_blocked", + ): + result = await unchain_url("http://169.254.169.254/latest/meta-data/") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_redirect_to_internal_blocked(self) -> None: + """외부 URL이 내부 IP로 리다이렉트할 때 두 번째 hop에서 차단.""" + safety_call_count = 0 + + async def _safety_per_hop(url: str) -> str | None: + nonlocal safety_call_count + safety_call_count += 1 + # 첫 번째 hop은 통과, 두 번째(내부 IP)는 차단 + if "internal" in url or "127.0.0.1" in url: + return "ssrf_blocked" + return None + + responses = [ + _make_response(302, {"location": "http://127.0.0.1/secret"}), + ] + client = _mock_client(responses) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch( + "app.services.unchainer.unchain._check_host_safety", + side_effect=_safety_per_hop, + ), + ): + result = await unchain_url("https://evil.com/redirect") + + assert result.error == "ssrf_blocked" + assert "ssrf_blocked" in result.signals + + @pytest.mark.asyncio + async def test_public_url_passes(self) -> None: + """공개 IP는 정상 통과.""" + client = _mock_client([_make_response(200)]) + + with ( + patch(_PATCH_TARGET, return_value=client), + patch( + "app.services.unchainer.unchain._check_host_safety", + new_callable=AsyncMock, + return_value=None, + ), + ): + result = await unchain_url("https://example.com/") + + assert result.error is None + assert "ssrf_blocked" not in result.signals + + +class TestIsPrivateIp: + """_is_private_ip 단위 테스트.""" + + def test_loopback(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("127.0.0.1") is True + assert _is_private_ip("::1") is True + + def test_private_ranges(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("10.0.0.1") is True + assert _is_private_ip("172.16.0.1") is True + assert _is_private_ip("192.168.1.1") is True + + def test_link_local(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("169.254.169.254") is True + + def test_public_ip(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("8.8.8.8") is False + assert _is_private_ip("1.1.1.1") is False + + def test_invalid_ip(self) -> None: + from app.services.unchainer.unchain import _is_private_ip + + assert _is_private_ip("not-an-ip") is True From 1517f5072a7db34989976589bbfa0263575c3c57 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 13:29:35 +0900 Subject: [PATCH 030/173] =?UTF-8?q?[Chore]=20=EC=96=B8=EC=B2=B4=EC=9D=B4?= =?UTF-8?q?=EB=8B=9D=20=EB=8D=B0=EB=AA=A8=20bit.ly=20URL=20=EA=B5=90?= =?UTF-8?q?=EC=B2=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/demo_unchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/demo/demo_unchain.py b/tests/demo/demo_unchain.py index d527838..018e505 100644 --- a/tests/demo/demo_unchain.py +++ b/tests/demo/demo_unchain.py @@ -16,7 +16,7 @@ CASES: list[tuple[str, str]] = [ ("리다이렉트 없음", "https://www.google.com/"), ("HTTP → HTTPS 업그레이드", "http://google.com/"), - ("단축 URL (bit.ly 예시)", "https://bit.ly/3kF8Yv2"), + ("단축 URL (bit.ly 사용)", "https://bit.ly/4tGFJkf"), ("t.co 단축 URL", "https://t.co/example"), ("HTTP 301 리다이렉트", "http://github.com/"), ("상대 경로 리다이렉트", "https://httpbin.org/relative-redirect/2"), From 73773d70645d274e2a516d0b1d812f52f76f5605 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 13:48:53 +0900 Subject: [PATCH 031/173] =?UTF-8?q?[Chore]=20=EC=96=B8=EC=B2=B4=EC=9D=B4?= =?UTF-8?q?=EB=8B=9D=20=EB=8D=B0=EB=AA=A8=20=EC=A3=BC=EC=84=9D=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20=EB=B0=8F=20=EC=8B=A4=EC=A0=9C=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=20=EA=B0=80=EB=8A=A5=ED=95=9C=20URL=EB=A1=9C=20=EA=B5=90?= =?UTF-8?q?=EC=B2=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/demo_unchain.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/demo/demo_unchain.py b/tests/demo/demo_unchain.py index 018e505..8bc2b9e 100644 --- a/tests/demo/demo_unchain.py +++ b/tests/demo/demo_unchain.py @@ -14,13 +14,20 @@ # 실제 리다이렉트가 일어나는 공개 URL들 (데모 시점에 동작 보장 불가) CASES: list[tuple[str, str]] = [ - ("리다이렉트 없음", "https://www.google.com/"), - ("HTTP → HTTPS 업그레이드", "http://google.com/"), - ("단축 URL (bit.ly 사용)", "https://bit.ly/4tGFJkf"), - ("t.co 단축 URL", "https://t.co/example"), - ("HTTP 301 리다이렉트", "http://github.com/"), - ("상대 경로 리다이렉트", "https://httpbin.org/relative-redirect/2"), - ("절대 경로 리다이렉트", "https://httpbin.org/absolute-redirect/3"), + # --- 기본 동작 --- + ("리다이렉트 없음 (200 OK)", "https://www.google.com/"), + ("HTTP→HTTPS 영구 이동 (301)", "http://github.com/"), + ("단축 URL (bit.ly → 실제 목적지)", "https://bit.ly/4tGFJkf"), + # --- 301 vs 302 차이 --- + ("301 Permanent: 주소가 영구적으로 바뀜", "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=301"), + ("302 Found: 임시로 다른 곳으로 보냄", "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=302"), + # --- 다중 hop·경로 해석 --- + ("다중 hop 체인 (3회 리다이렉트)", "https://httpbin.org/redirect/3"), + ("상대 경로 Location 해석", "https://httpbin.org/relative-redirect/2"), + # --- 의심 신호 탐지 --- + ("스킴 다운그레이드 (HTTPS→HTTP)", "https://httpbin.org/redirect-to?url=http://httpbin.org/get&status_code=302"), + ("크로스 오리진 (호스트 변경)", "https://httpbin.org/redirect-to?url=https://www.google.com&status_code=302"), + # --- 에러 --- ("존재하지 않는 도메인 (DNS 실패)", "https://this-domain-does-not-exist-12345.com/"), ] From 6665444c562ef4547a8ed569cdb2c26af4c6b938 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 13 Apr 2026 14:00:42 +0900 Subject: [PATCH 032/173] =?UTF-8?q?[Chore]=20unchain=5Fmax=5Fhops=20?= =?UTF-8?q?=EA=B8=B0=EB=B3=B8=EA=B0=92=2020=20=E2=86=92=205=EB=A1=9C=20?= =?UTF-8?q?=EC=B6=95=EC=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/config.py b/app/core/config.py index bb3a2f2..ff981a8 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -78,7 +78,7 @@ def alembic_database_url(self) -> str: normalizer_max_url_length: int = 1024 # 언체이닝 - unchain_max_hops: int = 20 + unchain_max_hops: int = 5 unchain_timeout_seconds: float = 5.0 unchain_connect_timeout_seconds: float = 3.0 unchain_chain_timeout_seconds: float = 20.0 From 80aaa9af96d0ed9d4088b593ac54709d7ea9257e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 12:03:38 +0900 Subject: [PATCH 033/173] =?UTF-8?q?[Fix]=20HEAD=20=EB=A6=AC=EB=8B=A4?= =?UTF-8?q?=EC=9D=B4=EB=A0=89=ED=8A=B8=EC=97=90=20Location=20=EB=88=84?= =?UTF-8?q?=EB=9D=BD=20=EC=8B=9C=20GET=20=ED=8F=B4=EB=B0=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/unchain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index 7365cee..4b9544e 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -198,6 +198,9 @@ async def _follow_one_hop( if resp.status_code in _REDIRECT_CODES: raw_location = resp.headers.get("location") if not raw_location: + # 일부 서버는 HEAD 응답에서 Location을 생략하므로 GET으로 재시도 + if method == "HEAD": + continue hop = HopRecord( url=url, status_code=resp.status_code, method=method, ) From 5d035b6fe439841c8f2d29667f8ed0bd7f7b755a Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 12:04:44 +0900 Subject: [PATCH 034/173] =?UTF-8?q?[Test]=20=EC=8B=A0=EA=B7=9C=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=20test=5Fhead=5Fredirect=5Fwithout=5Flocatio?= =?UTF-8?q?n=5Ffalls=5Fback=5Fto=5Fget=20=EC=B6=94=EA=B0=80,=20=EA=B8=B0?= =?UTF-8?q?=EC=A1=B4=20test=5Fmissing=5Flocation=5Fheader=EB=8A=94=20HEAD/?= =?UTF-8?q?GET=20=EB=AA=A8=EB=91=90=20=EB=88=84=EB=9D=BD=ED=95=98=EB=8A=94?= =?UTF-8?q?=20=EC=BC=80=EC=9D=B4=EC=8A=A4=EB=A1=9C=20=EA=B0=B1=EC=8B=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/services/unchainer/test_unchain.py | 25 +++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py index f8e29db..8822d81 100644 --- a/tests/services/unchainer/test_unchain.py +++ b/tests/services/unchainer/test_unchain.py @@ -297,6 +297,28 @@ def _side_effect(method: str, *args: object, **kwargs: object) -> httpx.Response assert result.hops[0].method == "GET" assert result.error is None + @pytest.mark.asyncio + async def test_head_redirect_without_location_falls_back_to_get(self) -> None: + """HEAD가 리다이렉트인데 Location 누락 → GET으로 폴백해서 체인 추적.""" + + def _side_effect(method: str, url: str) -> httpx.Response: + if url == "https://example.com/start": + # HEAD는 301인데 Location 헤더 생략, GET은 정상 응답 + if method == "HEAD": + return _make_response(301) + return _make_response(301, {"location": "https://example.com/final"}) + return _make_response(200) + + client = _mock_client(side_effect=_side_effect) + + with patch(_PATCH_TARGET, return_value=client): + result = await unchain_url("https://example.com/start") + + assert result.error is None + assert result.final_url == "https://example.com/final" + assert result.hops[0].method == "GET" + assert result.hops[0].status_code == 301 + @pytest.mark.asyncio async def test_head_connect_error_falls_back_to_get(self) -> None: """HEAD에서 ConnectError → GET으로 폴백해서 성공.""" @@ -366,7 +388,8 @@ async def test_server_error_5xx(self) -> None: @pytest.mark.asyncio async def test_missing_location_header(self) -> None: - client = _mock_client([_make_response(302)]) + # HEAD·GET 모두 Location 헤더 누락 — 최종적으로 missing_location_header + client = _mock_client([_make_response(302), _make_response(302)]) with patch(_PATCH_TARGET, return_value=client): result = await unchain_url("https://broken.example.com/") From 043c0845a68f5ce5b768f59feb3f76328eace03e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:05:52 +0900 Subject: [PATCH 035/173] =?UTF-8?q?[Feat]=20=EC=99=B8=EB=B6=80=20=EC=9C=84?= =?UTF-8?q?=ED=98=91=20DB=20=EB=8C=80=EC=A1=B0=20=EC=84=9C=EB=B9=84?= =?UTF-8?q?=EC=8A=A4=20=EC=B6=94=EA=B0=80=20=E2=80=94=20GSB/URLhaus=20?= =?UTF-8?q?=EB=B3=91=EB=A0=AC=20=EC=A1=B0=ED=9A=8C,=20=EB=A7=A4=EC=B9=AD?= =?UTF-8?q?=20=ED=82=A4=20=EC=83=9D=EC=84=B1,=20CSV=20=EB=8F=99=EA=B8=B0?= =?UTF-8?q?=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/threat_db/__init__.py | 8 ++ app/services/threat_db/check.py | 66 +++++++++++ app/services/threat_db/gsb.py | 94 +++++++++++++++ app/services/threat_db/match_keys.py | 35 ++++++ app/services/threat_db/urlhaus.py | 72 +++++++++++ app/services/threat_db/urlhaus_sync.py | 158 +++++++++++++++++++++++++ 6 files changed, 433 insertions(+) create mode 100644 app/services/threat_db/__init__.py create mode 100644 app/services/threat_db/check.py create mode 100644 app/services/threat_db/gsb.py create mode 100644 app/services/threat_db/match_keys.py create mode 100644 app/services/threat_db/urlhaus.py create mode 100644 app/services/threat_db/urlhaus_sync.py diff --git a/app/services/threat_db/__init__.py b/app/services/threat_db/__init__.py new file mode 100644 index 0000000..45d287e --- /dev/null +++ b/app/services/threat_db/__init__.py @@ -0,0 +1,8 @@ +"""외부 위협 DB 대조 단계 (Stage 2). + +GSB(Google Safe Browsing) 실시간 조회와 URLhaus 로컬 스냅샷 조회를 병렬로 수행. +""" + +from app.services.threat_db.check import check_threat_db + +__all__ = ["check_threat_db"] diff --git a/app/services/threat_db/check.py b/app/services/threat_db/check.py new file mode 100644 index 0000000..a2826de --- /dev/null +++ b/app/services/threat_db/check.py @@ -0,0 +1,66 @@ +"""GSB + URLhaus 병렬 조회 후 결과 병합.""" + +from __future__ import annotations + +import asyncio + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.schemas.analysis import ( + GSBResult, + ThreatDbResult, + URLhausResult, +) +from app.services.threat_db.gsb import check_gsb +from app.services.threat_db.urlhaus import check_urlhaus + +logger = get_logger(__name__) + + +def _merge_threat_types(gsb: GSBResult, urlhaus: URLhausResult) -> list[str]: + seen: list[str] = [] + for m in gsb.matches: + if m.threat_type and m.threat_type not in seen: + seen.append(m.threat_type) + if urlhaus.threat and urlhaus.threat not in seen: + seen.append(urlhaus.threat) + return seen + + +async def check_threat_db(session: AsyncSession, final_url: str) -> ThreatDbResult: + """final_url 을 GSB + URLhaus 와 병렬 대조해 판정 결과 반환. + + 어느 한쪽이 실패해도 다른 쪽 결과로 판정한다. 두 쪽 다 실패 시 + is_malicious=False, sources_checked=0 로 반환하여 상위 레이어가 보수적으로 처리. + """ + gsb_task = check_gsb(final_url) + urlhaus_task = check_urlhaus(session, final_url) + + gsb_raw, urlhaus_raw = await asyncio.gather( + gsb_task, urlhaus_task, return_exceptions=True + ) + + if isinstance(gsb_raw, BaseException): + logger.warning("threat_db.gsb_unexpected", error=str(gsb_raw)) + gsb = GSBResult(checked=False, is_threat=False, error="unexpected") + else: + gsb = gsb_raw + + if isinstance(urlhaus_raw, BaseException): + logger.warning("threat_db.urlhaus_unexpected", error=str(urlhaus_raw)) + urlhaus = URLhausResult(checked=False, is_threat=False, error="unexpected") + else: + urlhaus = urlhaus_raw + + is_malicious = gsb.is_threat or urlhaus.is_threat + sources_checked = int(gsb.checked) + int(urlhaus.checked) + + return ThreatDbResult( + final_url=final_url, + is_malicious=is_malicious, + sources_checked=sources_checked, + gsb=gsb, + urlhaus=urlhaus, + threat_types=_merge_threat_types(gsb, urlhaus), + ) diff --git a/app/services/threat_db/gsb.py b/app/services/threat_db/gsb.py new file mode 100644 index 0000000..4d4f0d7 --- /dev/null +++ b/app/services/threat_db/gsb.py @@ -0,0 +1,94 @@ +"""Google Safe Browsing Lookup API 클라이언트. + +실패(타임아웃/인증/레이트리밋/서버 에러)는 모두 raise 하지 않고 +`GSBResult(checked=False, error=...)` 로 반환해 상위 파이프라인을 보호한다. +""" + +from __future__ import annotations + +import httpx + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.analysis import GSBMatch, GSBResult + +logger = get_logger(__name__) + +_LOGGED_MISSING_KEY = False + + +def _log_missing_key_once() -> None: + # API 키 미설정은 정상 경로이므로 로그 스팸 방지차 1회만 경고. + global _LOGGED_MISSING_KEY + if not _LOGGED_MISSING_KEY: + logger.warning("gsb.api_key_not_configured") + _LOGGED_MISSING_KEY = True + + +def _build_request_body(url: str) -> dict: + return { + "client": { + "clientId": settings.gsb_client_id, + "clientVersion": settings.gsb_client_version, + }, + "threatInfo": { + "threatTypes": [ + "MALWARE", + "SOCIAL_ENGINEERING", + "UNWANTED_SOFTWARE", + "POTENTIALLY_HARMFUL_APPLICATION", + ], + "platformTypes": ["ANY_PLATFORM"], + "threatEntryTypes": ["URL"], + "threatEntries": [{"url": url}], + }, + } + + +async def check_gsb(url: str) -> GSBResult: + """GSB Lookup API 호출 결과를 GSBResult 로 반환.""" + if not settings.gsb_api_key: + _log_missing_key_once() + return GSBResult(checked=False, is_threat=False, error="not_configured") + + params = {"key": settings.gsb_api_key} + body = _build_request_body(url) + + try: + async with httpx.AsyncClient(timeout=settings.gsb_timeout_seconds) as client: + resp = await client.post(settings.gsb_api_url, params=params, json=body) + except httpx.TimeoutException: + return GSBResult(checked=False, is_threat=False, error="timeout") + except httpx.HTTPError: + return GSBResult(checked=False, is_threat=False, error="http_error") + + status = resp.status_code + if status in (400, 401, 403): + return GSBResult(checked=False, is_threat=False, error="auth_error") + if status == 429: + return GSBResult(checked=False, is_threat=False, error="rate_limited") + if 500 <= status < 600: + return GSBResult(checked=False, is_threat=False, error=f"server_error_{status}") + if status != 200: + return GSBResult(checked=False, is_threat=False, error=f"http_error_{status}") + + try: + data = resp.json() + except ValueError: + return GSBResult(checked=False, is_threat=False, error="invalid_response") + + raw_matches = data.get("matches") or [] + matches = [ + GSBMatch( + threat_type=m.get("threatType", "UNKNOWN"), + platform_type=m.get("platformType"), + cache_duration=m.get("cacheDuration"), + ) + for m in raw_matches + ] + return GSBResult( + checked=True, + is_threat=bool(matches), + matches=matches, + error=None, + ) diff --git a/app/services/threat_db/match_keys.py b/app/services/threat_db/match_keys.py new file mode 100644 index 0000000..80c2c52 --- /dev/null +++ b/app/services/threat_db/match_keys.py @@ -0,0 +1,35 @@ +"""URLhaus 조회·동기화에 쓰는 매칭 키 생성. + +host 한 개가 기본이지만 GitHub/GitLab 같은 다중 테넌트 호스트는 +계정/리포 레벨에서 악성 여부가 갈리므로 host+path-prefix 키도 함께 생성. +""" + +from __future__ import annotations + +from urllib.parse import urlparse + +from app.core.config import settings + + +def derive_keys(url: str) -> list[str]: + """URL 에서 매칭 키 후보를 더 구체적인 순서로 반환. + + 반환: [host_path, host] 또는 [host] + host 추출 실패 시 빈 리스트. + """ + parsed = urlparse(url) + host = (parsed.hostname or "").lower() + if not host: + return [] + + required = settings.urlhaus_multitenant_hosts.get(host) + if required is None or required <= 0: + return [host] + + segments = [seg for seg in parsed.path.split("/") if seg] + if len(segments) < required: + return [host] + + prefix = "/".join(segments[:required]) + host_path = f"{host}/{prefix}" + return [host_path, host] diff --git a/app/services/threat_db/urlhaus.py b/app/services/threat_db/urlhaus.py new file mode 100644 index 0000000..6ae70d7 --- /dev/null +++ b/app/services/threat_db/urlhaus.py @@ -0,0 +1,72 @@ +"""URLhaus 로컬 SQLite 조회. + +URL 완전일치 → match_key(host_path/host) 조회 순. +""" + +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.logging import get_logger +from app.models.urlhaus_entry import URLhausEntry +from app.schemas.analysis import URLhausResult +from app.services.threat_db.match_keys import derive_keys + +logger = get_logger(__name__) + + +def _split_tags(tags: str | None) -> list[str]: + if not tags: + return [] + return [t.strip() for t in tags.split(",") if t.strip()] + + +def _to_result( + entry: URLhausEntry, + match_type: str, + matched_key: str, +) -> URLhausResult: + return URLhausResult( + checked=True, + is_threat=True, + match_type=match_type, # type: ignore[arg-type] + matched_key=matched_key, + threat=entry.threat, + tags=_split_tags(entry.tags), + urlhaus_link=entry.urlhaus_link, + ) + + +async def check_urlhaus(session: AsyncSession, url: str) -> URLhausResult: + """URLhaus 로컬 스냅샷에서 URL 매칭 여부 조회.""" + try: + # 1) URL 완전일치 + stmt = select(URLhausEntry).where(URLhausEntry.url == url) + row = (await session.execute(stmt)).scalar_one_or_none() + if row is not None: + return _to_result(row, "url", row.url) + + # 2) match_key IN (...) + keys = derive_keys(url) + if not keys: + return URLhausResult(checked=True, is_threat=False) + + stmt = select(URLhausEntry).where(URLhausEntry.match_key.in_(keys)) + rows = (await session.execute(stmt)).scalars().all() + if not rows: + return URLhausResult(checked=True, is_threat=False) + + # host_path(더 구체적) 우선 + by_key = {r.match_key: r for r in rows} + for key in keys: + hit = by_key.get(key) + if hit is not None: + match_type = "host_path" if "/" in key else "host" + return _to_result(hit, match_type, key) + + return URLhausResult(checked=True, is_threat=False) + except SQLAlchemyError as e: + logger.warning("urlhaus.db_error", error=str(e)) + return URLhausResult(checked=False, is_threat=False, error="db_error") diff --git a/app/services/threat_db/urlhaus_sync.py b/app/services/threat_db/urlhaus_sync.py new file mode 100644 index 0000000..34ca486 --- /dev/null +++ b/app/services/threat_db/urlhaus_sync.py @@ -0,0 +1,158 @@ +"""URLhaus CSV 다운로드 → 로컬 SQLite upsert. + +CSV 포맷(abuse.ch): + # comment lines starting with '#' + id,dateadded,url,url_status,last_online,threat,tags,urlhaus_link,reporter +""" + +from __future__ import annotations + +import csv +import io +from datetime import UTC, datetime +from urllib.parse import urlparse + +import httpx +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.exc import SQLAlchemyError + +from app.core.config import settings +from app.core.logging import get_logger +from app.db.session import SessionLocal +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db.match_keys import derive_keys + +logger = get_logger(__name__) + +_CSV_FIELDS = [ + "id", + "dateadded", + "url", + "url_status", + "last_online", + "threat", + "tags", + "urlhaus_link", + "reporter", +] + + +def _parse_dt(raw: str | None) -> datetime | None: + if not raw or raw in ("None", "N/A"): + return None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S UTC"): + try: + return datetime.strptime(raw.strip(), fmt).replace(tzinfo=UTC) + except ValueError: + continue + return None + + +def _derive_match_key(url: str) -> tuple[str, str] | None: + """(host, match_key) 반환. host 없으면 None.""" + keys = derive_keys(url) + if not keys: + return None + parsed = urlparse(url) + host = (parsed.hostname or "").lower() + # derive_keys 는 [host_path, host] 또는 [host] — 첫 원소가 가장 구체적 키. + return host, keys[0] + + +async def _fetch_csv() -> str | None: + try: + async with httpx.AsyncClient( + timeout=settings.urlhaus_download_timeout_seconds + ) as client: + resp = await client.get(settings.urlhaus_recent_csv_url) + except httpx.HTTPError as e: + logger.warning("urlhaus_sync.fetch_failed", error=str(e)) + return None + if resp.status_code != 200: + logger.warning("urlhaus_sync.fetch_bad_status", status=resp.status_code) + return None + return resp.text + + +def _iter_rows(csv_text: str): + # '#' 주석 라인 스킵 후 헤더 없는 CSV 로 파싱. + clean = io.StringIO( + "\n".join( + line for line in csv_text.splitlines() if line and not line.startswith("#") + ) + ) + reader = csv.reader(clean, quotechar='"', skipinitialspace=True) + for row in reader: + if len(row) < len(_CSV_FIELDS): + continue + yield dict(zip(_CSV_FIELDS, row[: len(_CSV_FIELDS)], strict=True)) + + +async def sync_urlhaus() -> dict: + """URLhaus CSV 를 동기화하고 {inserted, updated, total, failed} 통계 반환.""" + stats = {"inserted": 0, "updated": 0, "total": 0, "failed": 0} + + csv_text = await _fetch_csv() + if csv_text is None: + return stats + + now = datetime.now(UTC).replace(tzinfo=None) + + try: + async with SessionLocal() as session, session.begin(): + for raw in _iter_rows(csv_text): + stats["total"] += 1 + try: + entry_id = int(raw["id"]) + except (TypeError, ValueError): + stats["failed"] += 1 + continue + + url = raw.get("url") or "" + if not url: + stats["failed"] += 1 + continue + + derived = _derive_match_key(url) + if derived is None: + stats["failed"] += 1 + continue + host, match_key = derived + + values = { + "id": entry_id, + "url": url, + "host": host, + "match_key": match_key, + "threat": raw.get("threat") or None, + "tags": raw.get("tags") or None, + "url_status": raw.get("url_status") or None, + "date_added": _parse_dt(raw.get("dateadded")), + "last_online": _parse_dt(raw.get("last_online")), + "urlhaus_link": raw.get("urlhaus_link") or None, + "reporter": raw.get("reporter") or None, + "synced_at": now, + } + + stmt = sqlite_insert(URLhausEntry).values(**values) + update_cols = { + c: stmt.excluded[c] + for c in values + if c != "id" + } + stmt = stmt.on_conflict_do_update( + index_elements=["id"], set_=update_cols + ) + result = await session.execute(stmt) + # SQLite 는 upsert 시 rowcount 로 insert/update 구분이 애매 → + # lastrowid 유무로 추정. + if getattr(result, "is_insert", False) and result.inserted_primary_key: + stats["inserted"] += 1 + else: + stats["updated"] += 1 + except SQLAlchemyError as e: + logger.warning("urlhaus_sync.db_error", error=str(e)) + return stats + + logger.info("urlhaus_sync.completed", **stats) + return stats From e824463d9622799dd7b76d09835c3cfa51dac153 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:14:07 +0900 Subject: [PATCH 036/173] =?UTF-8?q?[Feat]=20URLhausEntry=20ORM=20=EB=AA=A8?= =?UTF-8?q?=EB=8D=B8=20=EB=B0=8F=20Alembic=20=EB=A7=88=EC=9D=B4=EA=B7=B8?= =?UTF-8?q?=EB=A0=88=EC=9D=B4=EC=85=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../versions/20260415_0610_urlhaus_entry.py | 61 +++++++++++++++++++ app/models/__init__.py | 3 +- app/models/urlhaus_entry.py | 36 +++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/20260415_0610_urlhaus_entry.py create mode 100644 app/models/urlhaus_entry.py diff --git a/alembic/versions/20260415_0610_urlhaus_entry.py b/alembic/versions/20260415_0610_urlhaus_entry.py new file mode 100644 index 0000000..e02ea89 --- /dev/null +++ b/alembic/versions/20260415_0610_urlhaus_entry.py @@ -0,0 +1,61 @@ +"""urlhaus_entry + +Revision ID: 818a3f82f172 +Revises: +Create Date: 2026-04-15 06:10:17.609945+00:00 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "818a3f82f172" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "urlhaus_entries", + sa.Column("id", sa.Integer(), autoincrement=False, nullable=False), + sa.Column("url", sa.String(), nullable=False), + sa.Column("host", sa.String(), nullable=False), + sa.Column("match_key", sa.String(), nullable=False), + sa.Column("threat", sa.String(), nullable=True), + sa.Column("tags", sa.String(), nullable=True), + sa.Column("url_status", sa.String(), nullable=True), + sa.Column("date_added", sa.DateTime(), nullable=True), + sa.Column("last_online", sa.DateTime(), nullable=True), + sa.Column("urlhaus_link", sa.String(), nullable=True), + sa.Column("reporter", sa.String(), nullable=True), + sa.Column("synced_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_urlhaus_entries")), + ) + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.create_index(batch_op.f("ix_urlhaus_entries_host"), ["host"], unique=False) + batch_op.create_index( + "ix_urlhaus_entries_host_match_key", ["host", "match_key"], unique=False + ) + batch_op.create_index( + batch_op.f("ix_urlhaus_entries_match_key"), ["match_key"], unique=False + ) + batch_op.create_index(batch_op.f("ix_urlhaus_entries_url"), ["url"], unique=True) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_url")) + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_match_key")) + batch_op.drop_index("ix_urlhaus_entries_host_match_key") + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_host")) + + op.drop_table("urlhaus_entries") + # ### end Alembic commands ### diff --git a/app/models/__init__.py b/app/models/__init__.py index 7015561..c0b7a2e 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,5 +4,6 @@ """ from app.db.base import Base +from app.models.urlhaus_entry import URLhausEntry -__all__ = ["Base"] +__all__ = ["Base", "URLhausEntry"] diff --git a/app/models/urlhaus_entry.py b/app/models/urlhaus_entry.py new file mode 100644 index 0000000..b3b5da9 --- /dev/null +++ b/app/models/urlhaus_entry.py @@ -0,0 +1,36 @@ +"""URLhaus 로컬 스냅샷 엔트리.""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import DateTime, Index, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class URLhausEntry(Base): + """URLhaus CSV 한 행을 upsert 한 레코드. + + `match_key` 는 host 또는 host+path-prefix 로, 조회 시 빠른 exact match 용도. + """ + + __tablename__ = "urlhaus_entries" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False) + url: Mapped[str] = mapped_column(String, nullable=False, unique=True, index=True) + host: Mapped[str] = mapped_column(String, nullable=False, index=True) + match_key: Mapped[str] = mapped_column(String, nullable=False, index=True) + threat: Mapped[str | None] = mapped_column(String, nullable=True) + tags: Mapped[str | None] = mapped_column(String, nullable=True) + url_status: Mapped[str | None] = mapped_column(String, nullable=True) + date_added: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + last_online: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + urlhaus_link: Mapped[str | None] = mapped_column(String, nullable=True) + reporter: Mapped[str | None] = mapped_column(String, nullable=True) + synced_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + __table_args__ = ( + Index("ix_urlhaus_entries_host_match_key", "host", "match_key"), + ) From 080e8b6820b96449450697d213bb90dc29d47411 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:14:37 +0900 Subject: [PATCH 037/173] =?UTF-8?q?[Feat]=20APScheduler=20=EC=8B=B1?= =?UTF-8?q?=EA=B8=80=ED=86=A4=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20lifespan?= =?UTF-8?q?=20=EC=97=90=20URLhaus=20=EC=9D=BC=EC=9D=BC=20=EB=8F=99?= =?UTF-8?q?=EA=B8=B0=ED=99=94=C2=B7=EB=B6=80=ED=8C=85=20=EC=A6=89=EC=8B=9C?= =?UTF-8?q?=20=EB=8F=99=EA=B8=B0=ED=99=94=20=ED=86=B5=ED=95=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/scheduler.py | 57 +++++++++++++++++++++++++++++++++++++++++++ app/main.py | 13 ++++++++++ 2 files changed, 70 insertions(+) create mode 100644 app/core/scheduler.py diff --git a/app/core/scheduler.py b/app/core/scheduler.py new file mode 100644 index 0000000..6861b4a --- /dev/null +++ b/app/core/scheduler.py @@ -0,0 +1,57 @@ +"""APScheduler 싱글톤 — URLhaus 일일 동기화 스케줄링.""" + +from __future__ import annotations + +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.cron import CronTrigger + +from app.core.config import settings +from app.core.logging import get_logger + +logger = get_logger(__name__) + +_scheduler: AsyncIOScheduler | None = None + + +def get_scheduler() -> AsyncIOScheduler: + global _scheduler + if _scheduler is None: + _scheduler = AsyncIOScheduler(timezone="UTC") + return _scheduler + + +def start_scheduler() -> None: + """URLhaus 동기화 job 을 등록하고 스케줄러 기동. + + `scheduler_enabled=False` 면 no-op (테스트 환경 용). + """ + if not settings.scheduler_enabled: + logger.info("scheduler.disabled") + return + + # import 시점에 순환 참조 방지차 지연 로드. + from app.services.threat_db.urlhaus_sync import sync_urlhaus + + scheduler = get_scheduler() + if scheduler.running: + return + + scheduler.add_job( + sync_urlhaus, + trigger=CronTrigger(hour=settings.urlhaus_sync_cron_hour, minute=0), + id="urlhaus_sync", + coalesce=True, + max_instances=1, + misfire_grace_time=3600, + replace_existing=True, + ) + scheduler.start() + logger.info("scheduler.started", cron_hour=settings.urlhaus_sync_cron_hour) + + +def shutdown_scheduler(wait: bool = False) -> None: + global _scheduler + if _scheduler is None or not _scheduler.running: + return + _scheduler.shutdown(wait=wait) + logger.info("scheduler.stopped") diff --git a/app/main.py b/app/main.py index 2b62a34..ccb2d82 100644 --- a/app/main.py +++ b/app/main.py @@ -1,3 +1,4 @@ +import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -8,6 +9,7 @@ from app.api.v1.router import api_router as api_v1_router from app.core.config import settings from app.core.logging import configure_logging, get_logger +from app.core.scheduler import shutdown_scheduler, start_scheduler from app.db.session import engine from app.middleware.request_context import RequestContextMiddleware @@ -22,10 +24,21 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: environment=settings.environment, version=settings.app_version, ) + start_scheduler() + startup_task: asyncio.Task | None = None + if settings.scheduler_enabled and settings.urlhaus_sync_on_startup: + # 최초 부트 시 즉시 1회 동기화 (백그라운드, 앱 기동은 블로킹하지 않음). + from app.services.threat_db.urlhaus_sync import sync_urlhaus + + startup_task = asyncio.create_task(sync_urlhaus()) + try: yield finally: logger.info("app.shutdown") + if startup_task is not None and not startup_task.done(): + startup_task.cancel() + shutdown_scheduler(wait=False) await engine.dispose() From 70b1f984377b3c0ea997c5574eadac24153a4388 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:15:11 +0900 Subject: [PATCH 038/173] =?UTF-8?q?[Feat]=20=EC=8A=A4=EC=BC=80=EC=A4=84?= =?UTF-8?q?=EB=9F=AC=C2=B7URLhaus=20=EB=8B=A4=EC=A4=91=20=ED=85=8C?= =?UTF-8?q?=EB=84=8C=ED=8A=B8=20=ED=98=B8=EC=8A=A4=ED=8A=B8=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95=EA=B3=BC=20apscheduler=20=EC=9D=98=EC=A1=B4=EC=84=B1?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 16 ++++++++++++++++ pyproject.toml | 1 + 2 files changed, 17 insertions(+) diff --git a/app/core/config.py b/app/core/config.py index ff981a8..24dd0e7 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -67,6 +67,22 @@ def alembic_database_url(self) -> str: urlhaus_online_csv_url: str = "https://urlhaus.abuse.ch/downloads/csv_online/" urlhaus_refresh_interval_seconds: int = 60 * 60 # 60 minutes urlhaus_download_timeout_seconds: float = 60.0 + # URLhaus 매칭 키: 다중 테넌트 호스트는 host+path-prefix(N 세그먼트)까지 키로 사용. + urlhaus_multitenant_hosts: dict[str, int] = Field( + default_factory=lambda: { + "github.com": 2, + "raw.githubusercontent.com": 2, + "gist.github.com": 2, + "gitlab.com": 2, + "bitbucket.org": 2, + "sites.google.com": 2, + } + ) + + # Scheduler / URLhaus 동기화 + scheduler_enabled: bool = True + urlhaus_sync_cron_hour: int = 3 # UTC + urlhaus_sync_on_startup: bool = True # RDAP rdap_bootstrap_url: str = "https://rdap.org/domain/" diff --git a/pyproject.toml b/pyproject.toml index c8dfd42..62ae0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "structlog>=24.4.0", "httpx>=0.28.0", "tenacity>=9.0.0", + "apscheduler>=3.10", ] [project.optional-dependencies] From b6d2c39ddd4b66a1dcefad14111f5ee6db087972 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:19:37 +0900 Subject: [PATCH 039/173] =?UTF-8?q?[Feat]=20GSBMatch/GSBResult/URLhausResu?= =?UTF-8?q?lt/ThreatDbResult=20=EC=8A=A4=ED=82=A4=EB=A7=88=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index a8e6826..7ce10bf 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import BaseModel, Field @@ -31,3 +33,44 @@ class UnchainResult(BaseModel): default_factory=list, description="탐지된 의심 신호 목록", ) + + +class GSBMatch(BaseModel): + """GSB Lookup 응답의 개별 위협 매치.""" + + threat_type: str = Field(description="MALWARE / SOCIAL_ENGINEERING 등") + platform_type: str | None = Field(default=None, description="ANY_PLATFORM 등") + cache_duration: str | None = Field(default=None, description="예: '300s'") + + +class GSBResult(BaseModel): + """GSB 조회 결과. 실패 시에도 raise 하지 않고 checked=False 로 반환.""" + + checked: bool = Field(description="실제 API 호출이 성공적으로 완료됐는지") + is_threat: bool = Field(default=False, description="하나 이상의 위협 매치 여부") + matches: list[GSBMatch] = Field(default_factory=list) + error: str | None = Field(default=None, description="실패 사유 코드") + + +class URLhausResult(BaseModel): + """URLhaus 로컬 스냅샷 조회 결과.""" + + checked: bool + is_threat: bool = False + match_type: Literal["url", "host", "host_path"] | None = None + matched_key: str | None = None + threat: str | None = None + tags: list[str] = Field(default_factory=list) + urlhaus_link: str | None = None + error: str | None = None + + +class ThreatDbResult(BaseModel): + """Stage 2 — GSB + URLhaus 대조 결과.""" + + final_url: str + is_malicious: bool + sources_checked: int = Field(description="성공적으로 조회된 외부 소스 수 (0~2)") + gsb: GSBResult + urlhaus: URLhausResult + threat_types: list[str] = Field(default_factory=list) From 04bbf57244a7913da1aa1468bbcabab8552f47c8 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:20:09 +0900 Subject: [PATCH 040/173] =?UTF-8?q?[Test]=20threat=5Fdb=20=EB=8B=A8?= =?UTF-8?q?=EC=9C=84=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EB=B0=8F=20=EA=B3=B5?= =?UTF-8?q?=EC=9A=A9=20async=5Fsession=C2=B7=EC=8A=A4=EC=BC=80=EC=A4=84?= =?UTF-8?q?=EB=9F=AC=20=EB=B9=84=ED=99=9C=EC=84=B1=20=ED=94=BD=EC=8A=A4?= =?UTF-8?q?=EC=B2=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 41 ++++++ tests/services/threat_db/__init__.py | 0 tests/services/threat_db/test_check.py | 94 ++++++++++++++ tests/services/threat_db/test_gsb.py | 121 ++++++++++++++++++ tests/services/threat_db/test_match_keys.py | 33 +++++ tests/services/threat_db/test_urlhaus.py | 110 ++++++++++++++++ tests/services/threat_db/test_urlhaus_sync.py | 95 ++++++++++++++ 7 files changed, 494 insertions(+) create mode 100644 tests/services/threat_db/__init__.py create mode 100644 tests/services/threat_db/test_check.py create mode 100644 tests/services/threat_db/test_gsb.py create mode 100644 tests/services/threat_db/test_match_keys.py create mode 100644 tests/services/threat_db/test_urlhaus.py create mode 100644 tests/services/threat_db/test_urlhaus_sync.py diff --git a/tests/conftest.py b/tests/conftest.py index dc723da..64b7e52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,42 @@ """공용 테스트 픽스처.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator + +# 모델 로드 (metadata 등록용) +import app.models # noqa: F401 +import pytest +import pytest_asyncio +from app.core.config import settings +from app.db.base import Base +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) + + +@pytest.fixture(autouse=True) +def _disable_scheduler(monkeypatch: pytest.MonkeyPatch) -> None: + """테스트에서는 APScheduler 기동 금지.""" + monkeypatch.setattr(settings, "scheduler_enabled", False) + + +@pytest_asyncio.fixture +async def async_session() -> AsyncIterator[AsyncSession]: + """인메모리 SQLite 세션 — 테스트마다 새 DB.""" + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + session_maker = async_sessionmaker( + bind=engine, expire_on_commit=False, autoflush=False + ) + async with session_maker() as session: + yield session + + await engine.dispose() diff --git a/tests/services/threat_db/__init__.py b/tests/services/threat_db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/threat_db/test_check.py b/tests/services/threat_db/test_check.py new file mode 100644 index 0000000..a2ba987 --- /dev/null +++ b/tests/services/threat_db/test_check.py @@ -0,0 +1,94 @@ +"""check_threat_db 병렬 조회·폴백 검증.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +from app.schemas.analysis import GSBMatch, GSBResult, URLhausResult +from app.services.threat_db.check import check_threat_db +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _run( + session: AsyncSession, + gsb: GSBResult, + urlhaus: URLhausResult, + url: str = "https://x.test/", +): + with patch( + "app.services.threat_db.check.check_gsb", AsyncMock(return_value=gsb) + ), patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=urlhaus), + ): + return await check_threat_db(session, url) + + +async def test_both_clean(async_session: AsyncSession) -> None: + result = await _run( + async_session, + GSBResult(checked=True, is_threat=False), + URLhausResult(checked=True, is_threat=False), + ) + assert result.is_malicious is False + assert result.sources_checked == 2 + assert result.threat_types == [] + + +async def test_gsb_hit_marks_malicious(async_session: AsyncSession) -> None: + gsb = GSBResult( + checked=True, + is_threat=True, + matches=[GSBMatch(threat_type="MALWARE")], + ) + result = await _run( + async_session, + gsb, + URLhausResult(checked=True, is_threat=False), + ) + assert result.is_malicious is True + assert "MALWARE" in result.threat_types + + +async def test_urlhaus_hit_with_gsb_error(async_session: AsyncSession) -> None: + """GSB 실패 + URLhaus 히트 → 여전히 malicious 판정.""" + result = await _run( + async_session, + GSBResult(checked=False, is_threat=False, error="timeout"), + URLhausResult( + checked=True, + is_threat=True, + match_type="host", + matched_key="evil.test", + threat="malware_download", + ), + ) + assert result.is_malicious is True + assert result.sources_checked == 1 + assert "malware_download" in result.threat_types + + +async def test_both_fail_returns_clean_zero_sources( + async_session: AsyncSession, +) -> None: + result = await _run( + async_session, + GSBResult(checked=False, is_threat=False, error="timeout"), + URLhausResult(checked=False, is_threat=False, error="db_error"), + ) + assert result.is_malicious is False + assert result.sources_checked == 0 + + +async def test_exception_is_caught(async_session: AsyncSession) -> None: + with patch( + "app.services.threat_db.check.check_gsb", + AsyncMock(side_effect=RuntimeError("boom")), + ), patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + ): + result = await check_threat_db(async_session, "https://x.test/") + assert result.gsb.checked is False + assert result.gsb.error == "unexpected" + assert result.sources_checked == 1 diff --git a/tests/services/threat_db/test_gsb.py b/tests/services/threat_db/test_gsb.py new file mode 100644 index 0000000..c00114f --- /dev/null +++ b/tests/services/threat_db/test_gsb.py @@ -0,0 +1,121 @@ +"""GSB check_gsb 단위 테스트 — httpx 모킹으로 각 실패 분기 검증.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.core.config import settings +from app.services.threat_db import gsb as gsb_module +from app.services.threat_db.gsb import check_gsb + + +@pytest.fixture(autouse=True) +def _with_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "gsb_api_key", "test-key") + # "1회 경고" 전역 플래그 초기화 — 키 미설정 테스트 영향 제거. + monkeypatch.setattr(gsb_module, "_LOGGED_MISSING_KEY", False) + + +def _mock_post_response(resp: httpx.Response) -> AsyncMock: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(return_value=resp) + return client + + +async def test_no_api_key_returns_not_configured(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "gsb_api_key", None) + result = await check_gsb("https://example.com") + assert result.checked is False + assert result.error == "not_configured" + + +async def test_match_returns_is_threat() -> None: + body = { + "matches": [ + { + "threatType": "MALWARE", + "platformType": "ANY_PLATFORM", + "cacheDuration": "300s", + } + ] + } + resp = httpx.Response(200, json=body, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://evil.test/") + assert result.checked is True + assert result.is_threat is True + assert result.matches[0].threat_type == "MALWARE" + + +async def test_no_match_returns_safe() -> None: + resp = httpx.Response(200, json={}, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://safe.test/") + assert result.checked is True + assert result.is_threat is False + assert result.matches == [] + + +async def test_timeout_mapped_to_error() -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(side_effect=httpx.TimeoutException("timeout")) + with patch("app.services.threat_db.gsb.httpx.AsyncClient", return_value=client): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "timeout" + + +@pytest.mark.parametrize( + "status,expected", + [ + (400, "auth_error"), + (401, "auth_error"), + (403, "auth_error"), + (429, "rate_limited"), + (500, "server_error_500"), + (503, "server_error_503"), + ], +) +async def test_http_status_errors(status: int, expected: str) -> None: + resp = httpx.Response(status, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == expected + + +async def test_invalid_json() -> None: + resp = httpx.Response(200, content=b"not json", request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "invalid_response" + + +async def test_generic_http_error() -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.post = AsyncMock(side_effect=httpx.HTTPError("boom")) + with patch("app.services.threat_db.gsb.httpx.AsyncClient", return_value=client): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "http_error" diff --git a/tests/services/threat_db/test_match_keys.py b/tests/services/threat_db/test_match_keys.py new file mode 100644 index 0000000..0a0a785 --- /dev/null +++ b/tests/services/threat_db/test_match_keys.py @@ -0,0 +1,33 @@ +"""match_keys.derive_keys 단위 테스트.""" + +from __future__ import annotations + +from app.services.threat_db.match_keys import derive_keys + + +def test_host_only_for_standard_domain() -> None: + assert derive_keys("https://example.com/path/deep") == ["example.com"] + + +def test_github_returns_host_path_and_host() -> None: + keys = derive_keys("https://github.com/alice/repo/blob/main/x.exe") + assert keys == ["github.com/alice/repo", "github.com"] + + +def test_raw_githubusercontent() -> None: + keys = derive_keys("https://raw.githubusercontent.com/alice/repo/main/x.sh") + assert keys[0] == "raw.githubusercontent.com/alice/repo" + assert keys[-1] == "raw.githubusercontent.com" + + +def test_github_short_path_fallbacks_to_host() -> None: + # path segment 가 2개 미만이면 host 만. + assert derive_keys("https://github.com/alice") == ["github.com"] + + +def test_empty_host_returns_empty_list() -> None: + assert derive_keys("not a url") == [] + + +def test_host_case_is_normalized() -> None: + assert derive_keys("https://Example.COM/x") == ["example.com"] diff --git a/tests/services/threat_db/test_urlhaus.py b/tests/services/threat_db/test_urlhaus.py new file mode 100644 index 0000000..b65d872 --- /dev/null +++ b/tests/services/threat_db/test_urlhaus.py @@ -0,0 +1,110 @@ +"""URLhaus 조회 단위 테스트 — 인메모리 SQLite 시딩.""" + +from __future__ import annotations + +from datetime import datetime +from unittest.mock import AsyncMock, patch + +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db.urlhaus import check_urlhaus +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _seed(session: AsyncSession, **kwargs) -> None: + defaults = { + "threat": "malware_download", + "tags": "exe,emotet", + "url_status": "online", + "date_added": datetime(2026, 4, 10, 0, 0, 0), + "last_online": None, + "urlhaus_link": "https://urlhaus.abuse.ch/url/1/", + "reporter": "tester", + "synced_at": datetime(2026, 4, 15, 0, 0, 0), + } + defaults.update(kwargs) + session.add(URLhausEntry(**defaults)) + await session.flush() + + +async def test_exact_url_match(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=1, + url="https://evil.test/malware.exe", + host="evil.test", + match_key="evil.test", + ) + result = await check_urlhaus(async_session, "https://evil.test/malware.exe") + assert result.is_threat is True + assert result.match_type == "url" + assert result.threat == "malware_download" + assert result.tags == ["exe", "emotet"] + + +async def test_host_match(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=2, + url="https://evil.test/other.bin", + host="evil.test", + match_key="evil.test", + ) + result = await check_urlhaus(async_session, "https://evil.test/something-else") + assert result.is_threat is True + assert result.match_type == "host" + assert result.matched_key == "evil.test" + + +async def test_host_path_match_github(async_session: AsyncSession) -> None: + await _seed( + async_session, + id=3, + url="https://github.com/bad/repo/raw/main/x.exe", + host="github.com", + match_key="github.com/bad/repo", + ) + result = await check_urlhaus( + async_session, "https://github.com/bad/repo/issues/1" + ) + assert result.is_threat is True + assert result.match_type == "host_path" + assert result.matched_key == "github.com/bad/repo" + + +async def test_host_path_preferred_over_host(async_session: AsyncSession) -> None: + # 같은 호스트에 host 키와 host_path 키가 모두 있으면 host_path 우선. + await _seed( + async_session, + id=4, + url="https://github.com/foo", + host="github.com", + match_key="github.com", + ) + await _seed( + async_session, + id=5, + url="https://github.com/bad/repo/a", + host="github.com", + match_key="github.com/bad/repo", + ) + result = await check_urlhaus( + async_session, "https://github.com/bad/repo/anything" + ) + assert result.match_type == "host_path" + + +async def test_no_match(async_session: AsyncSession) -> None: + result = await check_urlhaus(async_session, "https://clean.test/") + assert result.checked is True + assert result.is_threat is False + + +async def test_db_error_returns_checked_false(async_session: AsyncSession) -> None: + from sqlalchemy.exc import SQLAlchemyError + + with patch.object( + async_session, "execute", AsyncMock(side_effect=SQLAlchemyError("boom")) + ): + result = await check_urlhaus(async_session, "https://x.test/") + assert result.checked is False + assert result.error == "db_error" diff --git a/tests/services/threat_db/test_urlhaus_sync.py b/tests/services/threat_db/test_urlhaus_sync.py new file mode 100644 index 0000000..a3794e9 --- /dev/null +++ b/tests/services/threat_db/test_urlhaus_sync.py @@ -0,0 +1,95 @@ +"""URLhaus 동기화 단위 테스트 — 인메모리 SQLite + httpx 픽스처.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.db.base import Base +from app.models.urlhaus_entry import URLhausEntry +from app.services.threat_db import urlhaus_sync as sync_module +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +SAMPLE_CSV = """# URLhaus recent URLs +# id, dateadded, url, url_status, last_online, threat, tags, urlhaus_link, reporter +1,2026-04-14 00:00:00,https://evil.test/a.exe,online,,malware_download,"exe,emotet",https://urlhaus.abuse.ch/url/1/,tester +2,2026-04-14 00:05:00,https://github.com/bad/repo/raw/main/x.sh,online,,malware_download,"sh",https://urlhaus.abuse.ch/url/2/,tester +""" + + +@pytest.fixture +async def sync_engine_patch(monkeypatch: pytest.MonkeyPatch): + """SessionLocal 을 인메모리 DB 로 치환.""" + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + maker = async_sessionmaker(bind=engine, expire_on_commit=False) + monkeypatch.setattr(sync_module, "SessionLocal", maker) + yield maker + await engine.dispose() + + +def _mock_csv_response(text: str, status: int = 200) -> AsyncMock: + resp = httpx.Response( + status, text=text, request=httpx.Request("GET", "https://x/"), + ) + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.get = AsyncMock(return_value=resp) + return client + + +async def test_sync_inserts_rows(sync_engine_patch) -> None: + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(SAMPLE_CSV), + ): + stats = await sync_module.sync_urlhaus() + + assert stats["total"] == 2 + assert stats["inserted"] + stats["updated"] == 2 + assert stats["failed"] == 0 + + async with sync_engine_patch() as session: + rows = (await session.execute(select(URLhausEntry))).scalars().all() + assert len(rows) == 2 + by_id = {r.id: r for r in rows} + assert by_id[1].host == "evil.test" + assert by_id[1].match_key == "evil.test" + assert by_id[2].match_key == "github.com/bad/repo" + + +async def test_sync_idempotent(sync_engine_patch) -> None: + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(SAMPLE_CSV), + ): + await sync_module.sync_urlhaus() + stats2 = await sync_module.sync_urlhaus() + + # 재실행 시 row 수 유지 — upsert 가 insert 중복 없이 동작. + async with sync_engine_patch() as session: + rows = (await session.execute(select(URLhausEntry))).scalars().all() + assert len(rows) == 2 + assert stats2["total"] == 2 + + +async def test_sync_network_failure_returns_empty_stats( + sync_engine_patch, +) -> None: + client = AsyncMock() + client.__aenter__.return_value = client + client.__aexit__.return_value = None + client.get = AsyncMock(side_effect=httpx.HTTPError("net")) + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", return_value=client + ): + stats = await sync_module.sync_urlhaus() + assert stats["total"] == 0 + assert stats["inserted"] == 0 From 2edb25156ab231dc1aff7756ec39b2922055eed7 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:26:07 +0900 Subject: [PATCH 041/173] =?UTF-8?q?[Feat]=20=EC=99=B8=EB=B6=80=20=EC=9C=84?= =?UTF-8?q?=ED=98=91=20DB=20=EB=8C=80=EC=A1=B0=20=EB=8D=B0=EB=AA=A8=20?= =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/demo_threat_db.py | 101 +++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/demo/demo_threat_db.py diff --git a/tests/demo/demo_threat_db.py b/tests/demo/demo_threat_db.py new file mode 100644 index 0000000..8e8b387 --- /dev/null +++ b/tests/demo/demo_threat_db.py @@ -0,0 +1,101 @@ +"""외부 위협 DB 대조 데모. + +사용법: + alembic upgrade head # 최초 1회 + python -m tests.demo.demo_threat_db + +환경변수 `GSB_API_KEY` 가 설정돼 있으면 실제 GSB Lookup API 호출. +없으면 GSB 는 `checked=False, error=not_configured` 로 반환된다. + +URLhaus 는 로컬 SQLite 스냅샷 기반이므로 실행 첫 부분에서 1회 +`sync_urlhaus()` 를 호출해 CSV 를 수신한다. +""" + +from __future__ import annotations + +import asyncio + +from sqlalchemy import func, select + +from app.db.session import SessionLocal +from app.models.urlhaus_entry import URLhausEntry +from app.schemas.analysis import ThreatDbResult +from app.services.threat_db import check_threat_db +from app.services.threat_db.urlhaus_sync import sync_urlhaus + +DIVIDER = "-" * 72 + + +# GSB 공식 테스트 URL — 실제 악성이 아니지만 Lookup API 가 항상 MALWARE 로 +# 매치시켜 주는 시그니처. GSB_API_KEY 가 있어야 threat=True 로 반환된다. +GSB_TEST_URL = "http://malware.testing.google.test/testing/malware/" + + +async def _pick_urlhaus_sample() -> str | None: + """동기화된 DB 에서 URLhaus 등재 URL 하나를 랜덤 추출. + + 등재 URL 은 CSV 싱크 시점마다 달라지므로 하드코딩 불가 — 방금 받은 + 스냅샷에서 아무거나 하나 뽑아서 실제 매치가 일어나는지 시연한다. + """ + async with SessionLocal() as session: + stmt = select(URLhausEntry.url).order_by(func.random()).limit(1) + row = (await session.execute(stmt)).scalar_one_or_none() + return row + + +def _print_result(label: str, url: str, result: ThreatDbResult) -> None: + print(DIVIDER) + print(f"[{label}]") + print(f" input : {url}") + print(f" malicious : {result.is_malicious}") + print(f" sources_check : {result.sources_checked}") + print(f" threat_types : {result.threat_types}") + print( + f" gsb : checked={result.gsb.checked} " + f"threat={result.gsb.is_threat} error={result.gsb.error}" + ) + print( + f" urlhaus : checked={result.urlhaus.checked} " + f"threat={result.urlhaus.is_threat} " + f"match_type={result.urlhaus.match_type} " + f"matched_key={result.urlhaus.matched_key} " + f"error={result.urlhaus.error}" + ) + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Threat DB (GSB + URLhaus) Demo") + print("=" * 72) + print() + + print(">>> 1) URLhaus 스냅샷 동기화 (수 MB CSV 다운로드·파싱 — 실제 수십 초 소요)") + stats = await sync_urlhaus() + print(f" stats: {stats}") + print() + + urlhaus_sample = await _pick_urlhaus_sample() + + cases: list[tuple[str, str]] = [ + # 1) 둘 다 미매치 — 정상 URL + ("둘 다 미매치 (정상 URL)", "https://www.google.com/"), + # 2) 위협 DB 에 없는 평범한 샘플 도메인 + ("위험 DB 미등재 샘플 도메인", "https://example.com/"), + # 3) GSB 공식 테스트 URL — API 키 있으면 항상 MALWARE 매치 + ("GSB 공식 테스트 URL (MALWARE 매치)", GSB_TEST_URL), + ] + if urlhaus_sample: + cases.append(("URLhaus 로컬 스냅샷 등재 URL", urlhaus_sample)) + else: + print(" (URLhaus 샘플이 비어있어 URLhaus 매치 케이스는 건너뜀)") + + async with SessionLocal() as session: + for label, url in cases: + result = await check_threat_db(session, url) + _print_result(label, url, result) + print(DIVIDER) + + +if __name__ == "__main__": + asyncio.run(main()) From 65cf21bf1d5553b8b24aefa2162a40627a042577 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:27:26 +0900 Subject: [PATCH 042/173] =?UTF-8?q?[Docs]=20threat=5Fdb=20=EB=94=94?= =?UTF-8?q?=EB=A0=89=ED=86=A0=EB=A6=AC=20=EA=B5=AC=EC=A1=B0=C2=B7=EB=8F=99?= =?UTF-8?q?=EC=9E=91=C2=B7=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EB=AA=A9=EB=A1=9D?= =?UTF-8?q?=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a44a9ad..e1d1ea2 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,16 @@ linclean-fastapi/ │ ├── normalizer/ # 1단계: URL 정규화(Canonicalization) │ │ ├── __init__.py # 진입점 (normalize_url re-export) │ │ └── normalize.py # 입력 검증, 스킴·호스트 정규화, 포트·프래그먼트 제거, 퍼센트 인코딩 정돈, 경로 정규화, IDN 디코딩 -│ └── unchainer/ # 1-2단계: URL 언체이닝(리다이렉트 추적) -│ ├── __init__.py # 진입점 (unchain_url re-export) -│ └── unchain.py # HEAD+GET 폴백, 체인 총 timeout, 스킴 방어, 의심 신호 수집 +│ ├── unchainer/ # 1-2단계: URL 언체이닝(리다이렉트 추적) +│ │ ├── __init__.py # 진입점 (unchain_url re-export) +│ │ └── unchain.py # HEAD+GET 폴백, 체인 총 timeout, 스킴 방어, 의심 신호 수집 +│ └── threat_db/ # 2단계: 외부 위협 DB 대조 +│ ├── __init__.py # 진입점 (check_threat_db re-export) +│ ├── check.py # GSB + URLhaus 병렬 조회·병합 +│ ├── gsb.py # Google Safe Browsing Lookup API +│ ├── urlhaus.py # 로컬 SQLite 조회 +│ ├── urlhaus_sync.py # CSV 다운로드 → SQLite upsert +│ └── match_keys.py # URLhaus 매칭 키 생성 (host / host+path) │ ├── alembic/ # 외부 피드 캐시 스키마 마이그레이션 │ ├── env.py # SQLite + batch mode 설정 @@ -81,12 +88,19 @@ linclean-fastapi/ │ ├── conftest.py # 공용 픽스처 │ ├── demo/ # 데모 스크립트 │ │ ├── demo_normalize.py # URL 정규화 데모 -│ │ └── demo_unchain.py # URL 언체이닝 데모 +│ │ ├── demo_unchain.py # URL 언체이닝 데모 +│ │ └── demo_threat_db.py # 외부 위협 DB 대조 데모 │ └── services/ │ ├── normalizer/ -│ │ └── test_normalize.py # URL 정규화 단위 테스트 (38개) -│ └── unchainer/ -│ └── test_unchain.py # URL 언체이닝 단위 테스트 (23개) +│ │ └── test_normalize.py # URL 정규화 단위 테스트 +│ ├── unchainer/ +│ │ └── test_unchain.py # URL 언체이닝 단위 테스트 +│ └── threat_db/ +│ ├── test_match_keys.py # URLhaus 매칭 키 단위 테스트 +│ ├── test_gsb.py # GSB Lookup 단위 테스트 +│ ├── test_urlhaus.py # URLhaus 조회 단위 테스트 +│ ├── test_urlhaus_sync.py # URLhaus 동기화 단위 테스트 +│ └── test_check.py # 병렬 조회·판정·폴백 단위 테스트 │ ├── docs/ # 기획서 등 설계 문서 │ └── LinClean_기능_초안.pdf @@ -196,6 +210,24 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 중 하나라도 매치되면 그 자체로 강한 위험 신호이며, 점수 가산과 함께 후속 단계에 결과를 그대로 전달합니다. +**구현 노트 (`services/threat_db/`):** + +- **외부 의존성 실패는 파이프라인을 죽이지 않습니다.** GSB / URLhaus / DB / + 스케줄러 어디서 실패해도 `check_threat_db()` 는 항상 `ThreatDbResult` 를 + 반환하며, 실패 사유는 `error` 필드에 문자열 코드로 기록됩니다. GSB 만 실패한 + 경우 URLhaus 결과 단독으로 `is_malicious` 를 판정하고, 둘 다 실패하면 + `sources_checked=0` 으로 반환해 상위 레이어가 보수적으로 처리할 수 있게 합니다. +- **URLhaus 매칭 키**: 기본적으로 host 한 개를 키로 쓰되, GitHub / GitLab / + Bitbucket / sites.google.com 같은 다중 테넌트 호스트는 + `host + path-prefix(N 세그먼트)` 키를 추가로 생성합니다. 계정·리포 단위에서 + 악성 여부가 갈리는 도메인을 host 전체로 블랙리스트화해 오탐하지 않도록 합니다. + 동기화·조회 모두 동일한 `derive_keys()` 를 사용해 키 일관성을 보장합니다. +- **스케줄러**: `AsyncIOScheduler(timezone=UTC)` 싱글톤이 `urlhaus_sync` 를 + `CronTrigger(hour=urlhaus_sync_cron_hour, minute=0)` 로 1일 1회 실행합니다 + (`coalesce=True, max_instances=1, misfire_grace_time=3600`). 앱 부트 시 + `urlhaus_sync_on_startup=True` 이면 최초 1회 즉시 동기화를 백그라운드로 + 수행합니다. 테스트에서는 `settings.scheduler_enabled=False` 로 전역 비활성화. + ### 3단계 — 도메인 휴리스틱 분석 규칙 기반 점수표로 도메인의 위험 신호를 합산합니다. 도메인 등록 정보는 From b4fed9485018f1b772db0a929c6139925ac3cef8 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 15:28:38 +0900 Subject: [PATCH 043/173] =?UTF-8?q?[Chore]=20ruff=20import=20=EC=A0=95?= =?UTF-8?q?=EB=A0=AC=20=EC=9E=90=EB=8F=99=20=EC=88=98=EC=A0=95=20=EB=B0=98?= =?UTF-8?q?=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/unchain.py | 2 +- tests/demo/demo_normalize.py | 2 +- tests/services/unchainer/test_unchain.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index 4b9544e..887dfa3 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -67,7 +67,7 @@ async def unchain_url(url: str) -> UnchainResult: _unchain_url_inner(url), timeout=settings.unchain_chain_timeout_seconds, ) - except asyncio.TimeoutError: + except TimeoutError: return UnchainResult( input_url=url, final_url=url, diff --git a/tests/demo/demo_normalize.py b/tests/demo/demo_normalize.py index 4aa1c56..148544b 100644 --- a/tests/demo/demo_normalize.py +++ b/tests/demo/demo_normalize.py @@ -66,7 +66,7 @@ def main() -> None: normalize_url(raw) print(f" [{i:02d}] {desc}") print(f" input : {display_input}") - print(f" result : WARN -- expected error not raised!") + print(" result : WARN -- expected error not raised!") except NormalizationError as e: print(f" [{i:02d}] {desc}") print(f" input : {display_input}") diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py index 8822d81..9b4db8e 100644 --- a/tests/services/unchainer/test_unchain.py +++ b/tests/services/unchainer/test_unchain.py @@ -10,7 +10,6 @@ import httpx import pytest - from app.services.unchainer.unchain import unchain_url From ebbef5deb5ec4571f836d6266cdd93617b9982a2 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:49:14 +0900 Subject: [PATCH 044/173] =?UTF-8?q?[Fix]=20URLhaus=20=EB=8F=99=EA=B8=B0?= =?UTF-8?q?=ED=99=94=20=EC=B2=AD=ED=81=AC=20=EC=BB=A4=EB=B0=8B=20+=20inser?= =?UTF-8?q?t/update=20=ED=86=B5=EA=B3=84=20=EC=A0=95=ED=99=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/threat_db/urlhaus_sync.py | 197 ++++++++++++++++--------- 1 file changed, 129 insertions(+), 68 deletions(-) diff --git a/app/services/threat_db/urlhaus_sync.py b/app/services/threat_db/urlhaus_sync.py index 34ca486..115452a 100644 --- a/app/services/threat_db/urlhaus_sync.py +++ b/app/services/threat_db/urlhaus_sync.py @@ -3,16 +3,29 @@ CSV 포맷(abuse.ch): # comment lines starting with '#' id,dateadded,url,url_status,last_online,threat,tags,urlhaus_link,reporter + +설계 메모 +--------- +- **부분 진행 보존**: 전체 행을 단일 트랜잭션으로 감싸지 않고 `CHUNK_SIZE` 단위로 + 커밋한다. 중간에 DB 오류가 나도 직전 청크까지의 결과는 영속화되며, `stats` 는 + 실제로 커밋된 카운트만 보고한다. +- **insert/update 정확 분리**: SQLite 의 `INSERT ... ON CONFLICT DO UPDATE` 는 + cursor 반환 값으로 두 경로를 구분할 수 없다. 청크 처리 직전에 해당 청크의 id + 들이 이미 존재하는지 한 번 SELECT 해 결과를 분류한다. +- **시각 정규화**: SQLite 에는 모두 naive UTC 로 저장한다. `_parse_dt` 와 + `synced_at` 양쪽이 동일한 규약을 따라야 비교/조회가 깨지지 않는다. """ from __future__ import annotations import csv import io +from collections.abc import Iterable, Iterator from datetime import UTC, datetime from urllib.parse import urlparse import httpx +from sqlalchemy import select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import SQLAlchemyError @@ -36,13 +49,18 @@ "reporter", ] +# 청크 크기 — 너무 작으면 트랜잭션 오버헤드, 너무 크면 부분 진행 보존 효과 약화. +CHUNK_SIZE = 500 + def _parse_dt(raw: str | None) -> datetime | None: + """CSV 의 시각 문자열을 naive UTC datetime 으로 정규화.""" if not raw or raw in ("None", "N/A"): return None for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S UTC"): try: - return datetime.strptime(raw.strip(), fmt).replace(tzinfo=UTC) + # SQLite 컬럼이 naive 이므로 tzinfo 를 떼서 일관성 유지. + return datetime.strptime(raw.strip(), fmt).replace(tzinfo=None) except ValueError: continue return None @@ -61,9 +79,7 @@ def _derive_match_key(url: str) -> tuple[str, str] | None: async def _fetch_csv() -> str | None: try: - async with httpx.AsyncClient( - timeout=settings.urlhaus_download_timeout_seconds - ) as client: + async with httpx.AsyncClient(timeout=settings.urlhaus_download_timeout_seconds) as client: resp = await client.get(settings.urlhaus_recent_csv_url) except httpx.HTTPError as e: logger.warning("urlhaus_sync.fetch_failed", error=str(e)) @@ -74,22 +90,91 @@ async def _fetch_csv() -> str | None: return resp.text -def _iter_rows(csv_text: str): - # '#' 주석 라인 스킵 후 헤더 없는 CSV 로 파싱. +def _iter_raw_rows(csv_text: str) -> Iterator[list[str]]: + """CSV 의 데이터 행을 list 로 yield. 주석/빈 줄 스킵.""" clean = io.StringIO( - "\n".join( - line for line in csv_text.splitlines() if line and not line.startswith("#") - ) + "\n".join(line for line in csv_text.splitlines() if line and not line.startswith("#")) ) reader = csv.reader(clean, quotechar='"', skipinitialspace=True) - for row in reader: - if len(row) < len(_CSV_FIELDS): - continue - yield dict(zip(_CSV_FIELDS, row[: len(_CSV_FIELDS)], strict=True)) + yield from reader + + +def _build_values(raw: dict, now: datetime) -> dict | None: + """CSV 한 행을 ORM values dict 로 변환. 검증 실패 시 None.""" + try: + entry_id = int(raw["id"]) + except (TypeError, ValueError): + return None + + url = raw.get("url") or "" + if not url: + return None + + derived = _derive_match_key(url) + if derived is None: + return None + host, match_key = derived + + return { + "id": entry_id, + "url": url, + "host": host, + "match_key": match_key, + "threat": raw.get("threat") or None, + "tags": raw.get("tags") or None, + "url_status": raw.get("url_status") or None, + "date_added": _parse_dt(raw.get("dateadded")), + "last_online": _parse_dt(raw.get("last_online")), + "urlhaus_link": raw.get("urlhaus_link") or None, + "reporter": raw.get("reporter") or None, + "synced_at": now, + } + + +def _chunked(rows: Iterable[dict], size: int) -> Iterator[list[dict]]: + chunk: list[dict] = [] + for row in rows: + chunk.append(row) + if len(chunk) >= size: + yield chunk + chunk = [] + if chunk: + yield chunk + + +async def _flush_chunk(session, values_list: list[dict]) -> tuple[int, int]: + """한 청크를 upsert 하고 (inserted, updated) 카운트 반환. + + SQLite 의 `INSERT ... ON CONFLICT DO UPDATE` 는 cursor 결과로 insert/update + 를 구분할 수 없으므로, 청크 단위로 한 번 SELECT 해서 사전 분류한다. + """ + ids = [v["id"] for v in values_list] + existing_ids = set( + (await session.execute(select(URLhausEntry.id).where(URLhausEntry.id.in_(ids)))) + .scalars() + .all() + ) + + inserted = 0 + updated = 0 + for values in values_list: + stmt = sqlite_insert(URLhausEntry).values(**values) + update_cols = {c: stmt.excluded[c] for c in values if c != "id"} + stmt = stmt.on_conflict_do_update(index_elements=["id"], set_=update_cols) + await session.execute(stmt) + if values["id"] in existing_ids: + updated += 1 + else: + inserted += 1 + return inserted, updated async def sync_urlhaus() -> dict: - """URLhaus CSV 를 동기화하고 {inserted, updated, total, failed} 통계 반환.""" + """URLhaus CSV 를 동기화하고 {inserted, updated, total, failed} 통계 반환. + + `stats` 는 **실제 DB 에 커밋된 행 수** 만 누적한다. 중간 청크에서 오류가 나도 + 직전까지의 청크는 영속화되어 있고, 그만큼만 카운트된다. + """ stats = {"inserted": 0, "updated": 0, "total": 0, "failed": 0} csv_text = await _fetch_csv() @@ -98,61 +183,37 @@ async def sync_urlhaus() -> dict: now = datetime.now(UTC).replace(tzinfo=None) - try: - async with SessionLocal() as session, session.begin(): - for raw in _iter_rows(csv_text): - stats["total"] += 1 - try: - entry_id = int(raw["id"]) - except (TypeError, ValueError): - stats["failed"] += 1 - continue - - url = raw.get("url") or "" - if not url: - stats["failed"] += 1 - continue - - derived = _derive_match_key(url) - if derived is None: - stats["failed"] += 1 - continue - host, match_key = derived - - values = { - "id": entry_id, - "url": url, - "host": host, - "match_key": match_key, - "threat": raw.get("threat") or None, - "tags": raw.get("tags") or None, - "url_status": raw.get("url_status") or None, - "date_added": _parse_dt(raw.get("dateadded")), - "last_online": _parse_dt(raw.get("last_online")), - "urlhaus_link": raw.get("urlhaus_link") or None, - "reporter": raw.get("reporter") or None, - "synced_at": now, - } - - stmt = sqlite_insert(URLhausEntry).values(**values) - update_cols = { - c: stmt.excluded[c] - for c in values - if c != "id" - } - stmt = stmt.on_conflict_do_update( - index_elements=["id"], set_=update_cols + def _values_stream() -> Iterator[dict]: + for raw in _iter_raw_rows(csv_text): + stats["total"] += 1 + if len(raw) < len(_CSV_FIELDS): + stats["failed"] += 1 + continue + row_dict = dict(zip(_CSV_FIELDS, raw[: len(_CSV_FIELDS)], strict=True)) + values = _build_values(row_dict, now) + if values is None: + stats["failed"] += 1 + continue + yield values + + async with SessionLocal() as session: + for chunk in _chunked(_values_stream(), CHUNK_SIZE): + try: + async with session.begin(): + inserted, updated = await _flush_chunk(session, chunk) + stats["inserted"] += inserted + stats["updated"] += updated + except SQLAlchemyError as e: + logger.warning( + "urlhaus_sync.chunk_failed", + error=str(e), + inserted_so_far=stats["inserted"], + updated_so_far=stats["updated"], ) - result = await session.execute(stmt) - # SQLite 는 upsert 시 rowcount 로 insert/update 구분이 애매 → - # lastrowid 유무로 추정. - if getattr(result, "is_insert", False) and result.inserted_primary_key: - stats["inserted"] += 1 - else: - stats["updated"] += 1 - except SQLAlchemyError as e: - logger.warning("urlhaus_sync.db_error", error=str(e)) - return stats + # 실패한 청크는 롤백 — 그만큼은 retry 대상으로 failed 에 누적. + stats["failed"] += len(chunk) + # 후속 청크는 새 트랜잭션으로 계속 시도. + continue logger.info("urlhaus_sync.completed", **stats) return stats From 6eb5563e9dbd1504d578778c769de9fcc14cb5c8 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:49:51 +0900 Subject: [PATCH 045/173] =?UTF-8?q?[Fix]=20GSB=20=EC=8A=A4=ED=8E=99=20?= =?UTF-8?q?=EC=A0=95=EB=A0=AC=20=EB=B0=8F=20=EA=B2=BD=EA=B3=A0=20throttle?= =?UTF-8?q?=20=ED=82=A4=20=EA=B0=92=EB=B3=84=20=EA=B4=80=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/threat_db/gsb.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/app/services/threat_db/gsb.py b/app/services/threat_db/gsb.py index 4d4f0d7..741cac8 100644 --- a/app/services/threat_db/gsb.py +++ b/app/services/threat_db/gsb.py @@ -14,15 +14,22 @@ logger = get_logger(__name__) -_LOGGED_MISSING_KEY = False +# 키 값별로 1회만 경고. 운영 중 키가 토글되거나(hot reload) 테스트가 키를 +# 바꿔 끼우면 새 상태에 대해 다시 한 번 경고가 나간다. +_WARNED_KEY_STATES: set[str] = set() def _log_missing_key_once() -> None: - # API 키 미설정은 정상 경로이므로 로그 스팸 방지차 1회만 경고. - global _LOGGED_MISSING_KEY - if not _LOGGED_MISSING_KEY: - logger.warning("gsb.api_key_not_configured") - _LOGGED_MISSING_KEY = True + state = settings.gsb_api_key or "" + if state in _WARNED_KEY_STATES: + return + _WARNED_KEY_STATES.add(state) + logger.warning("gsb.api_key_not_configured") + + +def reset_missing_key_warning() -> None: + """테스트/hot-reload 시 throttle 상태 초기화.""" + _WARNED_KEY_STATES.clear() def _build_request_body(url: str) -> dict: @@ -70,7 +77,7 @@ async def check_gsb(url: str) -> GSBResult: if 500 <= status < 600: return GSBResult(checked=False, is_threat=False, error=f"server_error_{status}") if status != 200: - return GSBResult(checked=False, is_threat=False, error=f"http_error_{status}") + return GSBResult(checked=False, is_threat=False, error="http_error") try: data = resp.json() From 4668a7c601daf53967a0aaf99a882e7af02edb4e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:50:27 +0900 Subject: [PATCH 046/173] =?UTF-8?q?[Fix]=20threat=5Fdb=20=EB=B3=91?= =?UTF-8?q?=EB=A0=AC=20=EC=A1=B0=ED=9A=8C=EC=97=90=EC=84=9C=20CancelledErr?= =?UTF-8?q?or=20=EC=A0=84=ED=8C=8C=20=EB=B3=B4=EC=9E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/threat_db/check.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/services/threat_db/check.py b/app/services/threat_db/check.py index a2826de..acca66b 100644 --- a/app/services/threat_db/check.py +++ b/app/services/threat_db/check.py @@ -37,9 +37,13 @@ async def check_threat_db(session: AsyncSession, final_url: str) -> ThreatDbResu gsb_task = check_gsb(final_url) urlhaus_task = check_urlhaus(session, final_url) - gsb_raw, urlhaus_raw = await asyncio.gather( - gsb_task, urlhaus_task, return_exceptions=True - ) + gsb_raw, urlhaus_raw = await asyncio.gather(gsb_task, urlhaus_task, return_exceptions=True) + + # CancelledError 는 상위 task 의 취소 신호이므로 절대 삼키지 않는다. + # (shutdown / 요청 timeout 시 degraded 결과를 영속화하는 사고 방지) + for raw in (gsb_raw, urlhaus_raw): + if isinstance(raw, asyncio.CancelledError): + raise raw if isinstance(gsb_raw, BaseException): logger.warning("threat_db.gsb_unexpected", error=str(gsb_raw)) @@ -54,7 +58,7 @@ async def check_threat_db(session: AsyncSession, final_url: str) -> ThreatDbResu urlhaus = urlhaus_raw is_malicious = gsb.is_threat or urlhaus.is_threat - sources_checked = int(gsb.checked) + int(urlhaus.checked) + sources_checked = sum((gsb.checked, urlhaus.checked)) return ThreatDbResult( final_url=final_url, From 03da569b26c1e95dfffc49a2ac52228787ab8233 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:51:00 +0900 Subject: [PATCH 047/173] =?UTF-8?q?[Fix]=20URLhaus=20=EB=B6=80=ED=8A=B8=20?= =?UTF-8?q?=EB=8F=99=EA=B8=B0=ED=99=94=20task=20=EC=88=98=EB=AA=85=20?= =?UTF-8?q?=EA=B4=80=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/main.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index ccb2d82..8e7a79e 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -32,12 +32,32 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: startup_task = asyncio.create_task(sync_urlhaus()) + def _log_startup_sync_result(t: asyncio.Task) -> None: + # 백그라운드 task 의 예외가 사일런트하게 사라지지 않도록 로깅. + if t.cancelled(): + return + exc = t.exception() + if exc is not None: + logger.warning( + "urlhaus_sync.startup_failed", + error=str(exc), + error_type=type(exc).__name__, + ) + + startup_task.add_done_callback(_log_startup_sync_result) + try: yield finally: logger.info("app.shutdown") if startup_task is not None and not startup_task.done(): startup_task.cancel() + # cancel 후 짧게 대기해 "Task was destroyed but it is pending" 경고 방지. + with suppress(TimeoutError, asyncio.CancelledError): + await asyncio.wait_for( + asyncio.shield(asyncio.gather(startup_task, return_exceptions=True)), + timeout=2.0, + ) shutdown_scheduler(wait=False) await engine.dispose() From 268af0f37074929c829fb83ed7c2027ee4e5157e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:52:03 +0900 Subject: [PATCH 048/173] =?UTF-8?q?[Test]=20threat=5Fdb=20=EB=A6=AC?= =?UTF-8?q?=EB=B7=B0=20findings=20=ED=9A=8C=EA=B7=80=20=ED=85=8C=EC=8A=A4?= =?UTF-8?q?=ED=8A=B8=20=EB=B3=B4=EA=B0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/services/threat_db/test_check.py | 44 ++++++++++++++----- tests/services/threat_db/test_gsb.py | 29 +++++++++++- tests/services/threat_db/test_urlhaus_sync.py | 35 ++++++++++++--- 3 files changed, 89 insertions(+), 19 deletions(-) diff --git a/tests/services/threat_db/test_check.py b/tests/services/threat_db/test_check.py index a2ba987..8bd58fd 100644 --- a/tests/services/threat_db/test_check.py +++ b/tests/services/threat_db/test_check.py @@ -2,8 +2,10 @@ from __future__ import annotations +import asyncio from unittest.mock import AsyncMock, patch +import pytest from app.schemas.analysis import GSBMatch, GSBResult, URLhausResult from app.services.threat_db.check import check_threat_db from sqlalchemy.ext.asyncio import AsyncSession @@ -15,11 +17,12 @@ async def _run( urlhaus: URLhausResult, url: str = "https://x.test/", ): - with patch( - "app.services.threat_db.check.check_gsb", AsyncMock(return_value=gsb) - ), patch( - "app.services.threat_db.check.check_urlhaus", - AsyncMock(return_value=urlhaus), + with ( + patch("app.services.threat_db.check.check_gsb", AsyncMock(return_value=gsb)), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=urlhaus), + ), ): return await check_threat_db(session, url) @@ -81,14 +84,33 @@ async def test_both_fail_returns_clean_zero_sources( async def test_exception_is_caught(async_session: AsyncSession) -> None: - with patch( - "app.services.threat_db.check.check_gsb", - AsyncMock(side_effect=RuntimeError("boom")), - ), patch( - "app.services.threat_db.check.check_urlhaus", - AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + with ( + patch( + "app.services.threat_db.check.check_gsb", + AsyncMock(side_effect=RuntimeError("boom")), + ), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + ), ): result = await check_threat_db(async_session, "https://x.test/") assert result.gsb.checked is False assert result.gsb.error == "unexpected" assert result.sources_checked == 1 + + +async def test_cancelled_error_propagates(async_session: AsyncSession) -> None: + # 상위 shutdown/timeout 신호(CancelledError)는 절대 삼키지 않는다. + with ( + patch( + "app.services.threat_db.check.check_gsb", + AsyncMock(side_effect=asyncio.CancelledError()), + ), + patch( + "app.services.threat_db.check.check_urlhaus", + AsyncMock(return_value=URLhausResult(checked=True, is_threat=False)), + ), + pytest.raises(asyncio.CancelledError), + ): + await check_threat_db(async_session, "https://x.test/") diff --git a/tests/services/threat_db/test_gsb.py b/tests/services/threat_db/test_gsb.py index c00114f..1b26f30 100644 --- a/tests/services/threat_db/test_gsb.py +++ b/tests/services/threat_db/test_gsb.py @@ -14,8 +14,8 @@ @pytest.fixture(autouse=True) def _with_api_key(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(settings, "gsb_api_key", "test-key") - # "1회 경고" 전역 플래그 초기화 — 키 미설정 테스트 영향 제거. - monkeypatch.setattr(gsb_module, "_LOGGED_MISSING_KEY", False) + # "키 값별 1회 경고" throttle 초기화 — 테스트 간 상태 누수 방지. + gsb_module.reset_missing_key_warning() def _mock_post_response(resp: httpx.Response) -> AsyncMock: @@ -110,6 +110,31 @@ async def test_invalid_json() -> None: assert result.error == "invalid_response" +async def test_unmapped_non_200_returns_http_error() -> None: + # 3xx/기타 비매핑 상태코드는 스펙대로 "http_error" 로 평탄화된다. + resp = httpx.Response(301, request=httpx.Request("POST", "x")) + with patch( + "app.services.threat_db.gsb.httpx.AsyncClient", + return_value=_mock_post_response(resp), + ): + result = await check_gsb("https://x.test/") + assert result.checked is False + assert result.error == "http_error" + + +async def test_missing_key_warning_throttled_per_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 같은 키 상태로 두 번 호출해도 경고는 1회. 키 값이 바뀌면 다시 1회. + warnings: list[str] = [] + monkeypatch.setattr(settings, "gsb_api_key", None) + gsb_module.reset_missing_key_warning() + monkeypatch.setattr(gsb_module.logger, "warning", lambda event, **kw: warnings.append(event)) + await check_gsb("https://x.test/") + await check_gsb("https://y.test/") + assert warnings.count("gsb.api_key_not_configured") == 1 + + async def test_generic_http_error() -> None: client = AsyncMock() client.__aenter__.return_value = client diff --git a/tests/services/threat_db/test_urlhaus_sync.py b/tests/services/threat_db/test_urlhaus_sync.py index a3794e9..8d99735 100644 --- a/tests/services/threat_db/test_urlhaus_sync.py +++ b/tests/services/threat_db/test_urlhaus_sync.py @@ -36,7 +36,9 @@ async def sync_engine_patch(monkeypatch: pytest.MonkeyPatch): def _mock_csv_response(text: str, status: int = 200) -> AsyncMock: resp = httpx.Response( - status, text=text, request=httpx.Request("GET", "https://x/"), + status, + text=text, + request=httpx.Request("GET", "https://x/"), ) client = AsyncMock() client.__aenter__.return_value = client @@ -52,8 +54,10 @@ async def test_sync_inserts_rows(sync_engine_patch) -> None: ): stats = await sync_module.sync_urlhaus() + # 최초 실행 — 모두 insert, update 는 0 이어야 한다(C1 회귀 방지). assert stats["total"] == 2 - assert stats["inserted"] + stats["updated"] == 2 + assert stats["inserted"] == 2 + assert stats["updated"] == 0 assert stats["failed"] == 0 async with sync_engine_patch() as session: @@ -73,11 +77,32 @@ async def test_sync_idempotent(sync_engine_patch) -> None: await sync_module.sync_urlhaus() stats2 = await sync_module.sync_urlhaus() - # 재실행 시 row 수 유지 — upsert 가 insert 중복 없이 동작. + # 두 번째 실행은 모두 update 여야 한다 — insert/update 분류가 맞는지 검증. async with sync_engine_patch() as session: rows = (await session.execute(select(URLhausEntry))).scalars().all() assert len(rows) == 2 assert stats2["total"] == 2 + assert stats2["inserted"] == 0 + assert stats2["updated"] == 2 + assert stats2["failed"] == 0 + + +async def test_sync_short_row_counts_failed(sync_engine_patch) -> None: + # 필드 수가 모자라는 행은 스킵하되 failed 로 카운트되어야 한다(I6). + bad_csv = ( + "# header comment\n" + "1,2026-04-14 00:00:00,https://evil.test/a.exe\n" # 필드 부족 + '2,2026-04-14 00:05:00,https://good.test/b,online,,malware_download,"",' + "https://urlhaus.abuse.ch/url/2/,tester\n" + ) + with patch( + "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", + return_value=_mock_csv_response(bad_csv), + ): + stats = await sync_module.sync_urlhaus() + assert stats["total"] == 2 + assert stats["failed"] == 1 + assert stats["inserted"] == 1 async def test_sync_network_failure_returns_empty_stats( @@ -87,9 +112,7 @@ async def test_sync_network_failure_returns_empty_stats( client.__aenter__.return_value = client client.__aexit__.return_value = None client.get = AsyncMock(side_effect=httpx.HTTPError("net")) - with patch( - "app.services.threat_db.urlhaus_sync.httpx.AsyncClient", return_value=client - ): + with patch("app.services.threat_db.urlhaus_sync.httpx.AsyncClient", return_value=client): stats = await sync_module.sync_urlhaus() assert stats["total"] == 0 assert stats["inserted"] == 0 From c4ab2bfae0d56dff982503258db42f47a52718e7 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:55:07 +0900 Subject: [PATCH 049/173] =?UTF-8?q?[Docs]=20threat=5Fdb=20=EB=A6=AC?= =?UTF-8?q?=EB=B7=B0=20=ED=9B=84=EC=86=8D=20=E2=80=94=20=EC=B2=AD=ED=81=AC?= =?UTF-8?q?=20=EC=BB=A4=EB=B0=8B=C2=B7CancelledError=20=EC=84=A4=EA=B3=84?= =?UTF-8?q?=20=EB=85=B8=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 +++++++++++ tests/demo/demo_threat_db.py | 5 +++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e1d1ea2..2523588 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,17 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 (`coalesce=True, max_instances=1, misfire_grace_time=3600`). 앱 부트 시 `urlhaus_sync_on_startup=True` 이면 최초 1회 즉시 동기화를 백그라운드로 수행합니다. 테스트에서는 `settings.scheduler_enabled=False` 로 전역 비활성화. +- **청크 커밋 동기화**: `sync_urlhaus()` 는 CSV 수만 행을 단일 트랜잭션으로 + 감싸지 않고 `CHUNK_SIZE=500` 단위로 나눠 커밋합니다. 중간 청크에서 DB 오류가 + 나도 직전 청크까지의 결과는 영속화되며, 실패 청크는 롤백되어 `failed` 로 + 누적됩니다. `stats = {inserted, updated, total, failed}` 는 실제 커밋된 + 행 수만 반영하므로 재시도 대상 판정에 그대로 쓸 수 있습니다. insert/update + 분류는 청크 직전에 `SELECT` 로 기존 id 를 조회해 정확히 분리합니다 + (SQLite `ON CONFLICT DO UPDATE` 는 cursor 로 두 경로 구분 불가). +- **CancelledError 전파**: `check_threat_db()` 내부 `asyncio.gather(..., + return_exceptions=True)` 는 일반 예외만 degraded 결과로 흡수하고, + `CancelledError` 는 그대로 re-raise 합니다. 상위 shutdown / 요청 타임아웃 + 신호를 삼키면 degraded 결과가 영속화될 위험이 있기 때문입니다. ### 3단계 — 도메인 휴리스틱 분석 diff --git a/tests/demo/demo_threat_db.py b/tests/demo/demo_threat_db.py index 8e8b387..3cffa38 100644 --- a/tests/demo/demo_threat_db.py +++ b/tests/demo/demo_threat_db.py @@ -15,13 +15,12 @@ import asyncio -from sqlalchemy import func, select - from app.db.session import SessionLocal from app.models.urlhaus_entry import URLhausEntry from app.schemas.analysis import ThreatDbResult from app.services.threat_db import check_threat_db from app.services.threat_db.urlhaus_sync import sync_urlhaus +from sqlalchemy import func, select DIVIDER = "-" * 72 @@ -71,6 +70,8 @@ async def main() -> None: print() print(">>> 1) URLhaus 스냅샷 동기화 (수 MB CSV 다운로드·파싱 — 실제 수십 초 소요)") + # stats = {inserted, updated, total, failed} — 청크(500행) 단위 커밋이라 + # 중간에 깨져도 직전 청크까지는 영속화되고 그만큼만 집계된다. stats = await sync_urlhaus() print(f" stats: {stats}") print() From e0e3504828a90a2db473d52c3a2473a504982814 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:55:35 +0900 Subject: [PATCH 050/173] =?UTF-8?q?[Chore]=20.env.example=20=EC=8A=A4?= =?UTF-8?q?=EC=BC=80=EC=A4=84=EB=9F=AC=20=EC=84=A4=EC=A0=95=20=EB=85=B8?= =?UTF-8?q?=EC=B6=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..28f9fa1 --- /dev/null +++ b/.env.example @@ -0,0 +1,51 @@ +# ---- App ---- +APP_NAME="LinClean Open API" +APP_VERSION="0.1.0" +ENVIRONMENT=local +DEBUG=true + +# ---- Server ---- +HOST=0.0.0.0 +PORT=8000 +LOG_LEVEL=INFO +LOG_JSON=false + +# ---- CORS (JSON list) ---- +CORS_ORIGINS=["http://localhost:3000"] + +# ---- Database (SQLite) ---- +# Relative paths are resolved against the project root. +SQLITE_PATH=data/linclean.db +DB_ECHO=false + +# ---- Google Safe Browsing ---- +# Leave empty to disable GSB lookups in /api/v1/check. +GSB_API_KEY=SECRET_KEY +GSB_CLIENT_ID=linclean +GSB_CLIENT_VERSION=0.1.0 +GSB_TIMEOUT_SECONDS=5.0 + +# ---- URLhaus ---- +URLHAUS_RECENT_CSV_URL=https://urlhaus.abuse.ch/downloads/csv_recent/ +URLHAUS_ONLINE_CSV_URL=https://urlhaus.abuse.ch/downloads/csv_online/ +URLHAUS_REFRESH_INTERVAL_SECONDS=3600 +URLHAUS_DOWNLOAD_TIMEOUT_SECONDS=60.0 + +# ---- Scheduler / URLhaus sync ---- +# 스케줄러 켜면 매일 URLHAUS_SYNC_CRON_HOUR 시(UTC)에 CSV 동기화 실행. +# 부팅 시 1회 동기화는 URLHAUS_SYNC_ON_STARTUP 로 제어. 로컬/CI 에서 비활성화 가능. +SCHEDULER_ENABLED=true +URLHAUS_SYNC_CRON_HOUR=3 +URLHAUS_SYNC_ON_STARTUP=true + +# ---- RDAP ---- +RDAP_BOOTSTRAP_URL=https://rdap.org/domain/ +RDAP_TIMEOUT_SECONDS=5.0 +RDAP_CACHE_TTL_SECONDS=86400 +RDAP_NEW_DOMAIN_THRESHOLD_DAYS=30 + +# ---- Scoring ---- +SCORE_WEIGHT_GSB=50 +SCORE_WEIGHT_URLHAUS=50 +SCORE_WEIGHT_RDAP=20 +SCORE_MALICIOUS_THRESHOLD=50 From a18a586800d8e9e56b9f1f844462b2c1c0574266 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 20:56:06 +0900 Subject: [PATCH 051/173] =?UTF-8?q?[Chore]=20ruff=20format=20=EA=B3=B5?= =?UTF-8?q?=EB=B0=B1=C2=B7=EC=A4=84=EB=B0=94=EA=BF=88=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/error_handlers.py | 4 +--- app/core/config.py | 4 +--- app/models/urlhaus_entry.py | 4 +--- app/schemas/analysis.py | 4 +++- app/services/normalizer/normalize.py | 2 +- app/services/unchainer/unchain.py | 21 ++++++++++++++++----- tests/conftest.py | 4 +--- tests/demo/demo_unchain.py | 20 ++++++++++++++++---- tests/services/threat_db/test_urlhaus.py | 12 +++--------- tests/services/unchainer/test_unchain.py | 3 ++- 10 files changed, 45 insertions(+), 33 deletions(-) diff --git a/app/api/error_handlers.py b/app/api/error_handlers.py index 951b12e..210c920 100644 --- a/app/api/error_handlers.py +++ b/app/api/error_handlers.py @@ -30,9 +30,7 @@ async def app_error_handler(request: Request, exc: AppError) -> JSONResponse: ) @app.exception_handler(StarletteHTTPException) - async def http_exception_handler( - request: Request, exc: StarletteHTTPException - ) -> JSONResponse: + async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: return JSONResponse( status_code=exc.status_code, content=ErrorResponse( diff --git a/app/core/config.py b/app/core/config.py index 24dd0e7..c0506c1 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -55,9 +55,7 @@ def alembic_database_url(self) -> str: # Google Safe Browsing gsb_api_key: str | None = None - gsb_api_url: str = ( - "https://safebrowsing.googleapis.com/v4/threatMatches:find" - ) + gsb_api_url: str = "https://safebrowsing.googleapis.com/v4/threatMatches:find" gsb_client_id: str = "linclean" gsb_client_version: str = "0.1.0" gsb_timeout_seconds: float = 5.0 diff --git a/app/models/urlhaus_entry.py b/app/models/urlhaus_entry.py index b3b5da9..322aa42 100644 --- a/app/models/urlhaus_entry.py +++ b/app/models/urlhaus_entry.py @@ -31,6 +31,4 @@ class URLhausEntry(Base): reporter: Mapped[str | None] = mapped_column(String, nullable=True) synced_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) - __table_args__ = ( - Index("ix_urlhaus_entries_host_match_key", "host", "match_key"), - ) + __table_args__ = (Index("ix_urlhaus_entries_host_match_key", "host", "match_key"),) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 7ce10bf..238471a 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -15,7 +15,9 @@ class HopRecord(BaseModel): url: str = Field(description="이 hop에서 요청한 URL") status_code: int = Field(description="HTTP 응답 상태 코드") - raw_location: str | None = Field(default=None, description="Location 헤더 원본 값(상대경로 포함)") + raw_location: str | None = Field( + default=None, description="Location 헤더 원본 값(상대경로 포함)" + ) location: str | None = Field(default=None, description="절대경로로 해석된 Location 값") method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index c68bc57..d673630 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -36,7 +36,7 @@ def normalize_url(raw_url: str) -> NormalizeResult: # 길이 체크 전에 스킴 확정해야 함 — 스킴 붙이면 바이트 늘어남. # 단순 "://" 체크는 "example.com/path://weird" 같은 케이스 놓침. - if not re.match(r'^[a-zA-Z][a-zA-Z0-9+\-.]*://', cleaned): + if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", cleaned): cleaned = "https://" + cleaned max_len = settings.normalizer_max_url_length diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index 887dfa3..467fccb 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -48,7 +48,9 @@ async def _check_host_safety(url: str) -> str | None: try: addrinfo = await loop.getaddrinfo( - hostname, port, type=socket.SOCK_STREAM, + hostname, + port, + type=socket.SOCK_STREAM, ) except socket.gaierror: return "dns_failure" @@ -119,7 +121,10 @@ async def _unchain_url_inner(url: str) -> UnchainResult: break hop, next_url, hop_error = await _follow_one_hop( - client, current_url, headers, signals, + client, + current_url, + headers, + signals, ) if hop is not None: @@ -190,7 +195,9 @@ async def _follow_one_hop( # 5xx 서버 에러 → 체인 중단 if resp.status_code >= 500: hop = HopRecord( - url=url, status_code=resp.status_code, method=method, + url=url, + status_code=resp.status_code, + method=method, ) return hop, None, f"server_error_{resp.status_code}" @@ -202,7 +209,9 @@ async def _follow_one_hop( if method == "HEAD": continue hop = HopRecord( - url=url, status_code=resp.status_code, method=method, + url=url, + status_code=resp.status_code, + method=method, ) return hop, None, "missing_location_header" @@ -243,7 +252,9 @@ async def _follow_one_hop( # 리다이렉트 아닌 정상 응답 → 체인 종료 hop = HopRecord( - url=url, status_code=resp.status_code, method=method, + url=url, + status_code=resp.status_code, + method=method, ) return hop, None, None diff --git a/tests/conftest.py b/tests/conftest.py index 64b7e52..6b0b72a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,9 +33,7 @@ async def async_session() -> AsyncIterator[AsyncSession]: async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) - session_maker = async_sessionmaker( - bind=engine, expire_on_commit=False, autoflush=False - ) + session_maker = async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False) async with session_maker() as session: yield session diff --git a/tests/demo/demo_unchain.py b/tests/demo/demo_unchain.py index 8bc2b9e..c7b6ea9 100644 --- a/tests/demo/demo_unchain.py +++ b/tests/demo/demo_unchain.py @@ -19,14 +19,26 @@ ("HTTP→HTTPS 영구 이동 (301)", "http://github.com/"), ("단축 URL (bit.ly → 실제 목적지)", "https://bit.ly/4tGFJkf"), # --- 301 vs 302 차이 --- - ("301 Permanent: 주소가 영구적으로 바뀜", "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=301"), - ("302 Found: 임시로 다른 곳으로 보냄", "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=302"), + ( + "301 Permanent: 주소가 영구적으로 바뀜", + "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=301", + ), + ( + "302 Found: 임시로 다른 곳으로 보냄", + "https://httpbin.org/redirect-to?url=https://httpbin.org/get&status_code=302", + ), # --- 다중 hop·경로 해석 --- ("다중 hop 체인 (3회 리다이렉트)", "https://httpbin.org/redirect/3"), ("상대 경로 Location 해석", "https://httpbin.org/relative-redirect/2"), # --- 의심 신호 탐지 --- - ("스킴 다운그레이드 (HTTPS→HTTP)", "https://httpbin.org/redirect-to?url=http://httpbin.org/get&status_code=302"), - ("크로스 오리진 (호스트 변경)", "https://httpbin.org/redirect-to?url=https://www.google.com&status_code=302"), + ( + "스킴 다운그레이드 (HTTPS→HTTP)", + "https://httpbin.org/redirect-to?url=http://httpbin.org/get&status_code=302", + ), + ( + "크로스 오리진 (호스트 변경)", + "https://httpbin.org/redirect-to?url=https://www.google.com&status_code=302", + ), # --- 에러 --- ("존재하지 않는 도메인 (DNS 실패)", "https://this-domain-does-not-exist-12345.com/"), ] diff --git a/tests/services/threat_db/test_urlhaus.py b/tests/services/threat_db/test_urlhaus.py index b65d872..81f2ff5 100644 --- a/tests/services/threat_db/test_urlhaus.py +++ b/tests/services/threat_db/test_urlhaus.py @@ -63,9 +63,7 @@ async def test_host_path_match_github(async_session: AsyncSession) -> None: host="github.com", match_key="github.com/bad/repo", ) - result = await check_urlhaus( - async_session, "https://github.com/bad/repo/issues/1" - ) + result = await check_urlhaus(async_session, "https://github.com/bad/repo/issues/1") assert result.is_threat is True assert result.match_type == "host_path" assert result.matched_key == "github.com/bad/repo" @@ -87,9 +85,7 @@ async def test_host_path_preferred_over_host(async_session: AsyncSession) -> Non host="github.com", match_key="github.com/bad/repo", ) - result = await check_urlhaus( - async_session, "https://github.com/bad/repo/anything" - ) + result = await check_urlhaus(async_session, "https://github.com/bad/repo/anything") assert result.match_type == "host_path" @@ -102,9 +98,7 @@ async def test_no_match(async_session: AsyncSession) -> None: async def test_db_error_returns_checked_false(async_session: AsyncSession) -> None: from sqlalchemy.exc import SQLAlchemyError - with patch.object( - async_session, "execute", AsyncMock(side_effect=SQLAlchemyError("boom")) - ): + with patch.object(async_session, "execute", AsyncMock(side_effect=SQLAlchemyError("boom"))): result = await check_urlhaus(async_session, "https://x.test/") assert result.checked is False assert result.error == "db_error" diff --git a/tests/services/unchainer/test_unchain.py b/tests/services/unchainer/test_unchain.py index 9b4db8e..1463391 100644 --- a/tests/services/unchainer/test_unchain.py +++ b/tests/services/unchainer/test_unchain.py @@ -156,7 +156,8 @@ def _side_effect(*args: object, **kwargs: object) -> httpx.Response: nonlocal hop_count hop_count += 1 return _make_response( - 302, {"location": f"https://example.com/{hop_count}"}, + 302, + {"location": f"https://example.com/{hop_count}"}, ) client = _mock_client(side_effect=_side_effect) From 9379eb43265b6d1bcac7e5a7129c027dcb9e8e97 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Wed, 15 Apr 2026 21:43:23 +0900 Subject: [PATCH 052/173] =?UTF-8?q?[Fix]=20SQLAlchemy=EC=9D=98=20type=5Fan?= =?UTF-8?q?notation=5Fmap=EC=9D=84=20=EC=9C=84=ED=95=9C=20ClassVar=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/db/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/db/base.py b/app/db/base.py index e810128..6f93fcd 100644 --- a/app/db/base.py +++ b/app/db/base.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any +from typing import Any, ClassVar from sqlalchemy import MetaData from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -18,7 +18,7 @@ class Base(DeclarativeBase): metadata = MetaData(naming_convention=NAMING_CONVENTION) - type_annotation_map: dict[Any, Any] = {} + type_annotation_map: ClassVar[dict[Any, Any]] = {} class TimestampMixin: From 0a9cf454c796726b5ee8498d3d958ddec66ef0ad Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Thu, 16 Apr 2026 11:48:15 +0900 Subject: [PATCH 053/173] =?UTF-8?q?[Refactor]=20URLhaus=20=EB=8F=99?= =?UTF-8?q?=EA=B8=B0=ED=99=94=EB=A5=BC=20IntervalTrigger=20=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=84=ED=99=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 +-- README.md | 4 ++-- app/core/config.py | 1 - app/core/scheduler.py | 13 ++++++++----- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 28f9fa1..861a754 100644 --- a/.env.example +++ b/.env.example @@ -32,10 +32,9 @@ URLHAUS_REFRESH_INTERVAL_SECONDS=3600 URLHAUS_DOWNLOAD_TIMEOUT_SECONDS=60.0 # ---- Scheduler / URLhaus sync ---- -# 스케줄러 켜면 매일 URLHAUS_SYNC_CRON_HOUR 시(UTC)에 CSV 동기화 실행. +# 스케줄러 켜면 URLHAUS_REFRESH_INTERVAL_SECONDS 주기로 CSV 동기화 실행. # 부팅 시 1회 동기화는 URLHAUS_SYNC_ON_STARTUP 로 제어. 로컬/CI 에서 비활성화 가능. SCHEDULER_ENABLED=true -URLHAUS_SYNC_CRON_HOUR=3 URLHAUS_SYNC_ON_STARTUP=true # ---- RDAP ---- diff --git a/README.md b/README.md index 2523588..06b5f8d 100644 --- a/README.md +++ b/README.md @@ -223,8 +223,8 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 악성 여부가 갈리는 도메인을 host 전체로 블랙리스트화해 오탐하지 않도록 합니다. 동기화·조회 모두 동일한 `derive_keys()` 를 사용해 키 일관성을 보장합니다. - **스케줄러**: `AsyncIOScheduler(timezone=UTC)` 싱글톤이 `urlhaus_sync` 를 - `CronTrigger(hour=urlhaus_sync_cron_hour, minute=0)` 로 1일 1회 실행합니다 - (`coalesce=True, max_instances=1, misfire_grace_time=3600`). 앱 부트 시 + `IntervalTrigger(seconds=urlhaus_refresh_interval_seconds)` 로 주기 실행합니다 + (`coalesce=True, max_instances=1, misfire_grace_time=interval`). 앱 부트 시 `urlhaus_sync_on_startup=True` 이면 최초 1회 즉시 동기화를 백그라운드로 수행합니다. 테스트에서는 `settings.scheduler_enabled=False` 로 전역 비활성화. - **청크 커밋 동기화**: `sync_urlhaus()` 는 CSV 수만 행을 단일 트랜잭션으로 diff --git a/app/core/config.py b/app/core/config.py index c0506c1..4fb2bb9 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -79,7 +79,6 @@ def alembic_database_url(self) -> str: # Scheduler / URLhaus 동기화 scheduler_enabled: bool = True - urlhaus_sync_cron_hour: int = 3 # UTC urlhaus_sync_on_startup: bool = True # RDAP diff --git a/app/core/scheduler.py b/app/core/scheduler.py index 6861b4a..e1548f5 100644 --- a/app/core/scheduler.py +++ b/app/core/scheduler.py @@ -1,9 +1,9 @@ -"""APScheduler 싱글톤 — URLhaus 일일 동기화 스케줄링.""" +"""APScheduler 싱글톤 — URLhaus 주기 동기화 스케줄링.""" from __future__ import annotations from apscheduler.schedulers.asyncio import AsyncIOScheduler -from apscheduler.triggers.cron import CronTrigger +from apscheduler.triggers.interval import IntervalTrigger from app.core.config import settings from app.core.logging import get_logger @@ -38,15 +38,18 @@ def start_scheduler() -> None: scheduler.add_job( sync_urlhaus, - trigger=CronTrigger(hour=settings.urlhaus_sync_cron_hour, minute=0), + trigger=IntervalTrigger(seconds=settings.urlhaus_refresh_interval_seconds), id="urlhaus_sync", coalesce=True, max_instances=1, - misfire_grace_time=3600, + misfire_grace_time=settings.urlhaus_refresh_interval_seconds, replace_existing=True, ) scheduler.start() - logger.info("scheduler.started", cron_hour=settings.urlhaus_sync_cron_hour) + logger.info( + "scheduler.started", + interval_seconds=settings.urlhaus_refresh_interval_seconds, + ) def shutdown_scheduler(wait: bool = False) -> None: From fda953cba4aca37a8ab5988aa747f9d0e81ae025 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Thu, 16 Apr 2026 11:48:35 +0900 Subject: [PATCH 054/173] =?UTF-8?q?[Refactor]=20urlhaus=5Fentries=20?= =?UTF-8?q?=EC=9D=98=20=EB=AF=B8=EC=82=AC=EC=9A=A9=20=EC=9D=B8=EB=8D=B1?= =?UTF-8?q?=EC=8A=A4=20=EC=A0=95=EC=9D=98=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/models/urlhaus_entry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/models/urlhaus_entry.py b/app/models/urlhaus_entry.py index 322aa42..c9efb58 100644 --- a/app/models/urlhaus_entry.py +++ b/app/models/urlhaus_entry.py @@ -4,7 +4,7 @@ from datetime import datetime -from sqlalchemy import DateTime, Index, Integer, String +from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import Mapped, mapped_column from app.db.base import Base @@ -20,7 +20,7 @@ class URLhausEntry(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=False) url: Mapped[str] = mapped_column(String, nullable=False, unique=True, index=True) - host: Mapped[str] = mapped_column(String, nullable=False, index=True) + host: Mapped[str] = mapped_column(String, nullable=False) match_key: Mapped[str] = mapped_column(String, nullable=False, index=True) threat: Mapped[str | None] = mapped_column(String, nullable=True) tags: Mapped[str | None] = mapped_column(String, nullable=True) @@ -30,5 +30,3 @@ class URLhausEntry(Base): urlhaus_link: Mapped[str | None] = mapped_column(String, nullable=True) reporter: Mapped[str | None] = mapped_column(String, nullable=True) synced_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) - - __table_args__ = (Index("ix_urlhaus_entries_host_match_key", "host", "match_key"),) From 60df3ee66a7475d48bea465c01bb7288643d0741 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Thu, 16 Apr 2026 11:49:20 +0900 Subject: [PATCH 055/173] =?UTF-8?q?[Fix]=20urlhaus=5Fentries=20=EB=AF=B8?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=20=EC=9D=B8=EB=8D=B1=EC=8A=A4=20drop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../20260416_drop_unused_urlhaus_indexes.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 alembic/versions/20260416_drop_unused_urlhaus_indexes.py diff --git a/alembic/versions/20260416_drop_unused_urlhaus_indexes.py b/alembic/versions/20260416_drop_unused_urlhaus_indexes.py new file mode 100644 index 0000000..d25c66b --- /dev/null +++ b/alembic/versions/20260416_drop_unused_urlhaus_indexes.py @@ -0,0 +1,34 @@ +"""drop unused urlhaus indexes + +Revision ID: a1b2c3d4e5f6 +Revises: 818a3f82f172 +Create Date: 2026-04-16 00:00:00.000000+00:00 + +현재 조회 핫패스는 `url` 과 `match_key` 인덱스만 사용한다. `host` 단일 인덱스와 +`(host, match_key)` 복합 인덱스는 생성만 되고 참조되는 쿼리가 없어 쓰기 오버헤드만 +발생하므로 제거한다. +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: str | None = "818a3f82f172" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.drop_index("ix_urlhaus_entries_host_match_key") + batch_op.drop_index(batch_op.f("ix_urlhaus_entries_host")) + + +def downgrade() -> None: + with op.batch_alter_table("urlhaus_entries", schema=None) as batch_op: + batch_op.create_index(batch_op.f("ix_urlhaus_entries_host"), ["host"], unique=False) + batch_op.create_index( + "ix_urlhaus_entries_host_match_key", ["host", "match_key"], unique=False + ) From 4930ed7054bb28807b3aecf9e9325334483d5f67 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 20 Apr 2026 11:51:03 +0900 Subject: [PATCH 056/173] =?UTF-8?q?[Refactor]=20schemas/analysis.py?= =?UTF-8?q?=EB=A5=BC=20=EB=8F=84=EB=A9=94=EC=9D=B8=EB=B3=84=20=EB=AA=A8?= =?UTF-8?q?=EB=93=88=EB=A1=9C=20=EB=B6=84=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 107 +++++++++++---------------------------- app/schemas/analyze.py | 17 +++++++ app/schemas/normalize.py | 6 +++ app/schemas/pipeline.py | 41 +++++++++++++++ app/schemas/threat_db.py | 36 +++++++++++++ app/schemas/unchain.py | 19 +++++++ 6 files changed, 148 insertions(+), 78 deletions(-) create mode 100644 app/schemas/analyze.py create mode 100644 app/schemas/normalize.py create mode 100644 app/schemas/pipeline.py create mode 100644 app/schemas/threat_db.py create mode 100644 app/schemas/unchain.py diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 238471a..3302fcc 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -1,78 +1,29 @@ -from typing import Literal - -from pydantic import BaseModel, Field - - -class NormalizeResult(BaseModel): - """Stage 1 — URL canonicalization result.""" - - original_url: str = Field(description="Original URL after trimming") - normalized_url: str = Field(description="Canonicalized URL") - - -class HopRecord(BaseModel): - """리다이렉트 체인의 개별 hop.""" - - url: str = Field(description="이 hop에서 요청한 URL") - status_code: int = Field(description="HTTP 응답 상태 코드") - raw_location: str | None = Field( - default=None, description="Location 헤더 원본 값(상대경로 포함)" - ) - location: str | None = Field(default=None, description="절대경로로 해석된 Location 값") - method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") - - -class UnchainResult(BaseModel): - """Stage 2 — URL 언체이닝(리다이렉트 추적) 결과.""" - - input_url: str = Field(description="언체이닝 시작 URL") - final_url: str = Field(description="최종 도달 URL") - hops: list[HopRecord] = Field(default_factory=list, description="리다이렉트 hop 기록") - hop_count: int = Field(default=0, description="총 hop 수") - timed_out: bool = Field(default=False, description="타임아웃 발생 여부") - error: str | None = Field(default=None, description="체인 중단 사유(에러 발생 시)") - signals: list[str] = Field( - default_factory=list, - description="탐지된 의심 신호 목록", - ) - - -class GSBMatch(BaseModel): - """GSB Lookup 응답의 개별 위협 매치.""" - - threat_type: str = Field(description="MALWARE / SOCIAL_ENGINEERING 등") - platform_type: str | None = Field(default=None, description="ANY_PLATFORM 등") - cache_duration: str | None = Field(default=None, description="예: '300s'") - - -class GSBResult(BaseModel): - """GSB 조회 결과. 실패 시에도 raise 하지 않고 checked=False 로 반환.""" - - checked: bool = Field(description="실제 API 호출이 성공적으로 완료됐는지") - is_threat: bool = Field(default=False, description="하나 이상의 위협 매치 여부") - matches: list[GSBMatch] = Field(default_factory=list) - error: str | None = Field(default=None, description="실패 사유 코드") - - -class URLhausResult(BaseModel): - """URLhaus 로컬 스냅샷 조회 결과.""" - - checked: bool - is_threat: bool = False - match_type: Literal["url", "host", "host_path"] | None = None - matched_key: str | None = None - threat: str | None = None - tags: list[str] = Field(default_factory=list) - urlhaus_link: str | None = None - error: str | None = None - - -class ThreatDbResult(BaseModel): - """Stage 2 — GSB + URLhaus 대조 결과.""" - - final_url: str - is_malicious: bool - sources_checked: int = Field(description="성공적으로 조회된 외부 소스 수 (0~2)") - gsb: GSBResult - urlhaus: URLhausResult - threat_types: list[str] = Field(default_factory=list) +# 하위 호환 re-export. 신규 코드는 각 모듈에서 직접 import할 것. +from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import ( + PipelineFailure, + PipelineResult, + PipelineStage, + PipelineStages, + PipelineSuccess, +) +from app.schemas.threat_db import GSBMatch, GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import HopRecord, UnchainResult + +__all__ = [ + "AnalyzeAccepted", + "AnalyzeRequest", + "GSBMatch", + "GSBResult", + "HopRecord", + "NormalizeResult", + "PipelineFailure", + "PipelineResult", + "PipelineStage", + "PipelineStages", + "PipelineSuccess", + "ThreatDbResult", + "URLhausResult", + "UnchainResult", +] diff --git a/app/schemas/analyze.py b/app/schemas/analyze.py new file mode 100644 index 0000000..81dd1a2 --- /dev/null +++ b/app/schemas/analyze.py @@ -0,0 +1,17 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class AnalyzeRequest(BaseModel): + model_config = {"populate_by_name": True} + + analysis_id: str = Field(alias="analysisId") + url: str + + +class AnalyzeAccepted(BaseModel): + model_config = {"populate_by_name": True} + + analysis_id: str = Field(alias="analysisId") + status: Literal["queued"] = "queued" diff --git a/app/schemas/normalize.py b/app/schemas/normalize.py new file mode 100644 index 0000000..b19b4e8 --- /dev/null +++ b/app/schemas/normalize.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel, Field + + +class NormalizeResult(BaseModel): + original_url: str = Field(description="Original URL after trimming") + normalized_url: str = Field(description="Canonicalized URL") diff --git a/app/schemas/pipeline.py b/app/schemas/pipeline.py new file mode 100644 index 0000000..9a74ce4 --- /dev/null +++ b/app/schemas/pipeline.py @@ -0,0 +1,41 @@ +from enum import Enum +from typing import Literal + +from pydantic import BaseModel + +from app.schemas.normalize import NormalizeResult +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult + + +class PipelineStage(str, Enum): + NORMALIZE = "normalize" + UNCHAIN = "unchain" + THREAT_DB = "threat_db" + + +class PipelineStages(BaseModel): + """성공한 파이프라인의 각 단계 원시 결과. 모든 필드가 반드시 채워진다.""" + + normalize: NormalizeResult + unchain: UnchainResult + threat_db: ThreatDbResult + + +class PipelineSuccess(BaseModel): + status: Literal["success"] = "success" + analysis_id: str + original_url: str + final_url: str + stages: PipelineStages + + +class PipelineFailure(BaseModel): + status: Literal["failed"] = "failed" + analysis_id: str + original_url: str + failed_at_stage: PipelineStage + error: str + + +PipelineResult = PipelineSuccess | PipelineFailure diff --git a/app/schemas/threat_db.py b/app/schemas/threat_db.py new file mode 100644 index 0000000..ee9a960 --- /dev/null +++ b/app/schemas/threat_db.py @@ -0,0 +1,36 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class GSBMatch(BaseModel): + threat_type: str = Field(description="MALWARE / SOCIAL_ENGINEERING 등") + platform_type: str | None = None + cache_duration: str | None = None + + +class GSBResult(BaseModel): + checked: bool = Field(description="API 호출 성공 여부") + is_threat: bool = False + matches: list[GSBMatch] = Field(default_factory=list) + error: str | None = None + + +class URLhausResult(BaseModel): + checked: bool + is_threat: bool = False + match_type: Literal["url", "host", "host_path"] | None = None + matched_key: str | None = None + threat: str | None = None + tags: list[str] = Field(default_factory=list) + urlhaus_link: str | None = None + error: str | None = None + + +class ThreatDbResult(BaseModel): + final_url: str + is_malicious: bool + sources_checked: int = Field(description="성공적으로 조회된 외부 소스 수 (0~2)") + gsb: GSBResult + urlhaus: URLhausResult + threat_types: list[str] = Field(default_factory=list) diff --git a/app/schemas/unchain.py b/app/schemas/unchain.py new file mode 100644 index 0000000..75f0e14 --- /dev/null +++ b/app/schemas/unchain.py @@ -0,0 +1,19 @@ +from pydantic import BaseModel, Field + + +class HopRecord(BaseModel): + url: str = Field(description="이 hop에서 요청한 URL") + status_code: int = Field(description="HTTP 응답 상태 코드") + raw_location: str | None = Field(default=None, description="Location 헤더 원본 값") + location: str | None = Field(default=None, description="절대경로로 해석된 Location 값") + method: str = Field(default="HEAD", description="요청에 사용한 HTTP 메서드") + + +class UnchainResult(BaseModel): + input_url: str = Field(description="언체이닝 시작 URL") + final_url: str = Field(description="최종 도달 URL") + hops: list[HopRecord] = Field(default_factory=list) + hop_count: int = Field(default=0) + timed_out: bool = Field(default=False) + error: str | None = Field(default=None) + signals: list[str] = Field(default_factory=list) From b19e0562be5ea816c27e86c564534af66400709d Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 20 Apr 2026 11:51:36 +0900 Subject: [PATCH 057/173] =?UTF-8?q?[Refactor]=20pipeline=EC=9D=84=20?= =?UTF-8?q?=EB=8B=A8=EA=B3=84=EB=B3=84=20=ED=95=A8=EC=88=98=EB=A1=9C=20?= =?UTF-8?q?=EB=B6=84=EB=A6=AC=ED=95=98=EA=B3=A0=20=ED=8C=90=EB=B3=84=20?= =?UTF-8?q?=EC=9C=A0=EB=8B=88=EC=96=B8=20=EB=B0=98=ED=99=98=20=ED=83=80?= =?UTF-8?q?=EC=9E=85=20=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/pipeline.py | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 app/services/pipeline.py diff --git a/app/services/pipeline.py b/app/services/pipeline.py new file mode 100644 index 0000000..23c8c5d --- /dev/null +++ b/app/services/pipeline.py @@ -0,0 +1,71 @@ +"""URL 분석 파이프라인 오케스트레이터.""" + +from __future__ import annotations + +import structlog +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.exceptions import NormalizationError +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import ( + PipelineFailure, + PipelineStage, + PipelineStages, + PipelineSuccess, +) +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult +from app.services.normalizer import normalize_url +from app.services.threat_db import check_threat_db +from app.services.unchainer import unchain_url + +logger = structlog.get_logger(__name__) + + +def _stage_normalize(log: structlog.stdlib.BoundLogger, original_url: str) -> NormalizeResult: + result = normalize_url(original_url) + log.info("pipeline.normalize.done", normalized_url=result.normalized_url) + return result + + +async def _stage_unchain(log: structlog.stdlib.BoundLogger, normalized_url: str) -> UnchainResult: + result = await unchain_url(normalized_url) + log.info("pipeline.unchain.done", final_url=result.final_url, hops=result.hop_count, signals=result.signals) + return result + + +async def _stage_threat_db(log: structlog.stdlib.BoundLogger, final_url: str, session: AsyncSession) -> ThreatDbResult: + result = await check_threat_db(session, final_url) + log.info("pipeline.threat_db.done", is_malicious=result.is_malicious, sources_checked=result.sources_checked) + return result + + +async def run_pipeline( + analysis_id: str, + original_url: str, + session: AsyncSession, +) -> PipelineSuccess | PipelineFailure: + log = logger.bind(analysis_id=analysis_id) + log.info("pipeline.start", url=original_url) + + try: + norm = _stage_normalize(log, original_url) + except NormalizationError as exc: + log.warning("pipeline.failed", stage=PipelineStage.NORMALIZE, error=str(exc)) + return PipelineFailure( + analysis_id=analysis_id, + original_url=original_url, + failed_at_stage=PipelineStage.NORMALIZE, + error=str(exc), + ) + + unchain = await _stage_unchain(log, norm.normalized_url) + threat = await _stage_threat_db(log, unchain.final_url, session) + + log.info("pipeline.done", final_url=unchain.final_url) + return PipelineSuccess( + analysis_id=analysis_id, + original_url=original_url, + final_url=unchain.final_url, + stages=PipelineStages(normalize=norm, unchain=unchain, threat_db=threat), + ) From 2878b09d7ebfc616b66bfacdcf53a3d906fa057f Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 20 Apr 2026 11:53:18 +0900 Subject: [PATCH 058/173] =?UTF-8?q?[Feat]=20POST=20/api/v1/analyze=20?= =?UTF-8?q?=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EB=B0=8F=20dev?= =?UTF-8?q?=EC=9A=A9=20=EB=8F=99=EA=B8=B0=20=EC=97=94=EB=93=9C=ED=8F=AC?= =?UTF-8?q?=EC=9D=B8=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 5 ++++ app/api/deps.py | 16 ++++++++++++- app/api/v1/endpoints/analyze.py | 42 +++++++++++++++++++++++++++++++++ app/api/v1/endpoints/dev.py | 33 ++++++++++++++++++++++++++ app/api/v1/router.py | 9 ++++++- app/core/config.py | 4 ++++ 6 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 app/api/v1/endpoints/analyze.py create mode 100644 app/api/v1/endpoints/dev.py diff --git a/.env.example b/.env.example index 861a754..b43a378 100644 --- a/.env.example +++ b/.env.example @@ -48,3 +48,8 @@ SCORE_WEIGHT_GSB=50 SCORE_WEIGHT_URLHAUS=50 SCORE_WEIGHT_RDAP=20 SCORE_MALICIOUS_THRESHOLD=50 + +# ---- Spring 통신 ---- +# secrets.token_hex(32) 로 생성한 사전 공유 키. Spring 과 동일한 값 설정 필수. +INTERNAL_API_KEY=REPLACE_ME +SPRING_INTERNAL_URL=http://localhost:8080 diff --git a/app/api/deps.py b/app/api/deps.py index ac94185..2b50252 100644 --- a/app/api/deps.py +++ b/app/api/deps.py @@ -2,9 +2,23 @@ from typing import Annotated -from fastapi import Depends +from fastapi import Depends, Header, HTTPException, status from sqlalchemy.ext.asyncio import AsyncSession +from app.core.config import settings from app.db.session import get_db DBSession = Annotated[AsyncSession, Depends(get_db)] + + +async def verify_internal_api_key( + x_internal_api_key: Annotated[str | None, Header()] = None, +) -> None: + if x_internal_api_key != settings.internal_api_key: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing X-Internal-Api-Key", + ) + + +InternalApiKey = Annotated[None, Depends(verify_internal_api_key)] diff --git a/app/api/v1/endpoints/analyze.py b/app/api/v1/endpoints/analyze.py new file mode 100644 index 0000000..3c8acd4 --- /dev/null +++ b/app/api/v1/endpoints/analyze.py @@ -0,0 +1,42 @@ +"""POST /api/v1/analyze — Spring 으로부터 URL 분석 위임을 받는 엔드포인트.""" + +from __future__ import annotations + +from fastapi import APIRouter, BackgroundTasks, status + +from app.api.deps import InternalApiKey +from app.db.session import SessionLocal +from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.services.pipeline import run_pipeline + +router = APIRouter() + + +async def _run_pipeline_task(analysis_id: str, original_url: str) -> None: + """백그라운드 태스크 전용 래퍼. 세션을 독립적으로 열고 닫는다.""" + async with SessionLocal() as session: + await run_pipeline( + analysis_id=analysis_id, + original_url=original_url, + session=session, + ) + + +@router.post( + "/analyze", + status_code=status.HTTP_202_ACCEPTED, + response_model=AnalyzeAccepted, + response_model_by_alias=True, + summary="URL 분석 위임 접수", +) +async def analyze( + body: AnalyzeRequest, + background_tasks: BackgroundTasks, + _: InternalApiKey, +) -> AnalyzeAccepted: + background_tasks.add_task( + _run_pipeline_task, + analysis_id=body.analysis_id, + original_url=body.url, + ) + return AnalyzeAccepted(analysis_id=body.analysis_id) diff --git a/app/api/v1/endpoints/dev.py b/app/api/v1/endpoints/dev.py new file mode 100644 index 0000000..a4dd138 --- /dev/null +++ b/app/api/v1/endpoints/dev.py @@ -0,0 +1,33 @@ +"""개발용 엔드포인트 — debug=true 환경에서만 등록됨.""" + +from __future__ import annotations + +import uuid + +from fastapi import APIRouter +from pydantic import BaseModel + +from app.api.deps import DBSession +from app.schemas.pipeline import PipelineFailure, PipelineSuccess +from app.services.pipeline import run_pipeline + +router = APIRouter() + + +class DevAnalyzeRequest(BaseModel): + url: str + + +@router.post( + "/analyze", + response_model=PipelineSuccess | PipelineFailure, + summary="[Dev] URL 분석 — 동기 결과 반환", + description="파이프라인을 동기로 실행해 결과를 즉시 반환합니다. debug=true 환경 전용.", +) +async def dev_analyze(body: DevAnalyzeRequest, session: DBSession) -> PipelineSuccess | PipelineFailure: + analysis_id = str(uuid.uuid4()) + return await run_pipeline( + analysis_id=analysis_id, + original_url=body.url, + session=session, + ) diff --git a/app/api/v1/router.py b/app/api/v1/router.py index ed8ad49..d2108cd 100644 --- a/app/api/v1/router.py +++ b/app/api/v1/router.py @@ -1,6 +1,13 @@ from fastapi import APIRouter -from app.api.v1.endpoints import health +from app.api.v1.endpoints import analyze, health +from app.core.config import settings api_router = APIRouter() api_router.include_router(health.router, tags=["health"]) +api_router.include_router(analyze.router, tags=["analyze"]) + +if settings.debug: + from app.api.v1.endpoints import dev + + api_router.include_router(dev.router, prefix="/dev", tags=["dev"]) diff --git a/app/core/config.py b/app/core/config.py index 4fb2bb9..83353ae 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -107,6 +107,10 @@ def alembic_database_url(self) -> str: score_weight_rdap: int = 20 score_malicious_threshold: int = 50 + # Spring 통신 + internal_api_key: str + spring_internal_url: str = "http://localhost:8080" + @property def is_production(self) -> bool: return self.environment == "production" From 3eb0de28aeb0441392a80d2fb6da97200f63d1b4 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 20 Apr 2026 11:53:55 +0900 Subject: [PATCH 059/173] =?UTF-8?q?[Refactor]=20RequestContextMiddleware?= =?UTF-8?q?=EB=A5=BC=20=EC=88=9C=EC=88=98=20ASGI=EB=A1=9C=20=EC=A0=84?= =?UTF-8?q?=ED=99=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/middleware/request_context.py | 53 +++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/app/middleware/request_context.py b/app/middleware/request_context.py index 4f5679f..b526ca4 100644 --- a/app/middleware/request_context.py +++ b/app/middleware/request_context.py @@ -1,42 +1,61 @@ +from __future__ import annotations + import time import uuid +from collections.abc import Awaitable, Callable +from typing import Any import structlog -from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from starlette.requests import Request -from starlette.responses import Response +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send logger = structlog.get_logger(__name__) REQUEST_ID_HEADER = "X-Request-ID" -class RequestContextMiddleware(BaseHTTPMiddleware): - """Assign a request ID and emit a structured access log per request.""" +class RequestContextMiddleware: + """요청마다 X-Request-ID를 생성·전파하고 structlog에 바인딩한다. + + BaseHTTPMiddleware 대신 순수 ASGI로 구현해 응답 버퍼링 오버헤드를 제거한다. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return - async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: - request_id = request.headers.get(REQUEST_ID_HEADER) or str(uuid.uuid4()) + headers = dict(scope.get("headers", [])) + raw_id = headers.get(REQUEST_ID_HEADER.lower().encode()) + request_id = raw_id.decode() if raw_id else str(uuid.uuid4()) structlog.contextvars.clear_contextvars() structlog.contextvars.bind_contextvars( request_id=request_id, - method=request.method, - path=request.url.path, + method=scope.get("method", ""), + path=scope.get("path", ""), ) start = time.perf_counter() + status_code = 0 + + async def send_wrapper(message: Message) -> None: + nonlocal status_code + if message["type"] == "http.response.start": + status_code = message["status"] + mutable = MutableHeaders(scope=message) + mutable.append(REQUEST_ID_HEADER, request_id) + await send(message) + try: - response = await call_next(request) + await self.app(scope, receive, send_wrapper) except Exception: duration_ms = (time.perf_counter() - start) * 1000 logger.exception("request.failed", duration_ms=round(duration_ms, 2)) raise duration_ms = (time.perf_counter() - start) * 1000 - response.headers[REQUEST_ID_HEADER] = request_id - logger.info( - "request.completed", - status_code=response.status_code, - duration_ms=round(duration_ms, 2), - ) - return response + logger.info("request.completed", status_code=status_code, duration_ms=round(duration_ms, 2)) From 885078d2a9e5ab2d5653058557cf58e93cb6a49a Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Mon, 20 Apr 2026 11:56:02 +0900 Subject: [PATCH 060/173] =?UTF-8?q?[Refactor]=20schemas/analysis.py?= =?UTF-8?q?=EB=A5=BC=20=EB=8F=84=EB=A9=94=EC=9D=B8=EB=B3=84=20=EB=AA=A8?= =?UTF-8?q?=EB=93=88=EB=A1=9C=20=EB=B6=84=EB=A6=AC=20=ED=9B=84=20import=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/normalizer/normalize.py | 2 +- app/services/threat_db/check.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/services/normalizer/normalize.py b/app/services/normalizer/normalize.py index d673630..3e2df69 100644 --- a/app/services/normalizer/normalize.py +++ b/app/services/normalizer/normalize.py @@ -12,7 +12,7 @@ from app.core.config import settings from app.core.exceptions import NormalizationError -from app.schemas.analysis import NormalizeResult +from app.schemas.normalize import NormalizeResult _DEFAULT_PORTS: dict[str, int] = {"http": 80, "https": 443, "ftp": 21} _ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https", "ftp"}) diff --git a/app/services/threat_db/check.py b/app/services/threat_db/check.py index acca66b..2a7cbd8 100644 --- a/app/services/threat_db/check.py +++ b/app/services/threat_db/check.py @@ -7,11 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.logging import get_logger -from app.schemas.analysis import ( - GSBResult, - ThreatDbResult, - URLhausResult, -) +from app.schemas.threat_db import GSBResult, ThreatDbResult, URLhausResult from app.services.threat_db.gsb import check_gsb from app.services.threat_db.urlhaus import check_urlhaus From e66d613653a36c5ce65af6ad0cd0fa4d7f7ff601 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:02:55 +0900 Subject: [PATCH 061/173] =?UTF-8?q?=20[Feat]=20tldextract=20=EC=9D=98?= =?UTF-8?q?=EC=A1=B4=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 62ae0da..a8a64fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "httpx>=0.28.0", "tenacity>=9.0.0", "apscheduler>=3.10", + "tldextract>=5.1.0", ] [project.optional-dependencies] From 36afcd96c80f3e5393280d76145128b469161abd Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:03:21 +0900 Subject: [PATCH 062/173] =?UTF-8?q?[Feat]=20=EB=8F=84=EB=A9=94=EC=9D=B8=20?= =?UTF-8?q?=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20=EC=84=A4=EC=A0=95?= =?UTF-8?q?=EA=B0=92=20=EC=B6=94=EA=B0=80=20(=EC=A0=90=EC=88=98=20?= =?UTF-8?q?=EA=B0=80=EC=A4=91=EC=B9=98,=20RDAP=20=ED=83=80=EC=9E=84?= =?UTF-8?q?=EC=95=84=EC=9B=83/=EC=BA=90=EC=8B=9C=20TTL)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app/core/config.py b/app/core/config.py index 83353ae..a93af51 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -107,6 +107,33 @@ def alembic_database_url(self) -> str: score_weight_rdap: int = 20 score_malicious_threshold: int = 50 + # 도메인 휴리스틱 점수 + score_weight_ip_direct: int = 40 + score_weight_typo_domain: int = 40 + score_weight_punycode_idn: int = 35 + score_weight_no_https: int = 30 + score_weight_new_domain: int = 30 + score_weight_subdomain_overuse: int = 25 + score_weight_open_redirect_param: int = 20 + score_weight_hyphen_overuse: int = 20 + score_weight_suspicious_tld: int = 20 + score_weight_dga_like: int = 15 + score_weight_hosting_platform: int = 15 + + # 휴리스틱 점수 캡 — GSB/URLhaus(각 50) high-confidence 시그널을 합산이 압도하지 않도록 클램프 + domain_heuristic_score_cap: int = 80 + + # DGA 탐지 임계값 — 0.7이면 netflix/spotify/snapchat 등 긴 영어 브랜드 오탐 + dga_entropy_threshold: float = 3.5 + dga_consonant_ratio_threshold: float = 0.8 + + # 서브도메인 레이블 임계값 + subdomain_label_threshold: int = 4 + + # 하이픈/라벨 길이 임계값 — 20자 미만이면 standardchartered(17), samsungelectronics(18) 등 오탐 + hyphen_count_threshold: int = 3 + domain_label_length_threshold: int = 20 + # Spring 통신 internal_api_key: str spring_internal_url: str = "http://localhost:8080" From d5f2582915a0b3ca2d434c5b9b6692f1d20265d9 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:03:39 +0900 Subject: [PATCH 063/173] =?UTF-8?q?[Feat]=20=EB=8F=84=EB=A9=94=EC=9D=B8=20?= =?UTF-8?q?=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20=EC=8A=A4=ED=82=A4?= =?UTF-8?q?=EB=A7=88=20=EC=B6=94=EA=B0=80=20(DomainHeuristicResult/Signal,?= =?UTF-8?q?=20RdapInfo)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/domain_heuristic.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 app/schemas/domain_heuristic.py diff --git a/app/schemas/domain_heuristic.py b/app/schemas/domain_heuristic.py new file mode 100644 index 0000000..c030194 --- /dev/null +++ b/app/schemas/domain_heuristic.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from datetime import datetime +from enum import StrEnum + +from pydantic import BaseModel + + +class RdapInfo(BaseModel): + domain: str + registrar: str | None + created_date: datetime | None + expiry_date: datetime | None + domain_age_days: int | None + is_new_domain: bool + + +class DomainHeuristicSignal(StrEnum): + IP_DIRECT = "IP_DIRECT" + TYPO_DOMAIN = "TYPO_DOMAIN" + PUNYCODE_IDN = "PUNYCODE_IDN" + NEW_DOMAIN = "NEW_DOMAIN" + SUBDOMAIN_OVERUSE = "SUBDOMAIN_OVERUSE" + NO_HTTPS = "NO_HTTPS" + OPEN_REDIRECT_PARAM = "OPEN_REDIRECT_PARAM" + HYPHEN_OVERUSE = "HYPHEN_OVERUSE" + SUSPICIOUS_TLD = "SUSPICIOUS_TLD" + DGA_LIKE = "DGA_LIKE" + HOSTING_PLATFORM = "HOSTING_PLATFORM" + + +class DomainHeuristicResult(BaseModel): + domain: str + score: int + signals: list[DomainHeuristicSignal] + rdap: RdapInfo | None = None + rdap_error: str | None = None From 10e44d81b68cc9e088211f394243438b2a5b7edd Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:04:22 +0900 Subject: [PATCH 064/173] =?UTF-8?q?[Feat]=20=EB=8F=84=EB=A9=94=EC=9D=B8=20?= =?UTF-8?q?=ED=8C=A8=ED=84=B4=20=EA=B2=80=EC=82=AC=20=EC=B6=94=EA=B0=80=20?= =?UTF-8?q?(IP/=EC=9D=98=EC=8B=AC=20TLD/=EC=84=9C=EB=B8=8C=EB=8F=84?= =?UTF-8?q?=EB=A9=94=EC=9D=B8=20=EA=B3=BC=EB=8B=A4/=ED=95=98=EC=9D=B4?= =?UTF-8?q?=ED=94=88/=EC=98=A4=ED=94=88=20=EB=A6=AC=EB=8B=A4=EC=9D=B4?= =?UTF-8?q?=EB=A0=89=ED=8A=B8/=ED=98=B8=EC=8A=A4=ED=8C=85=20=ED=94=8C?= =?UTF-8?q?=EB=9E=AB=ED=8F=BC)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/domain_heuristic/__init__.py | 3 + app/services/domain_heuristic/patterns.py | 147 ++++++++++++++++++ tests/services/domain_heuristic/__init__.py | 0 .../domain_heuristic/test_patterns.py | 112 +++++++++++++ 4 files changed, 262 insertions(+) create mode 100644 app/services/domain_heuristic/__init__.py create mode 100644 app/services/domain_heuristic/patterns.py create mode 100644 tests/services/domain_heuristic/__init__.py create mode 100644 tests/services/domain_heuristic/test_patterns.py diff --git a/app/services/domain_heuristic/__init__.py b/app/services/domain_heuristic/__init__.py new file mode 100644 index 0000000..f77c8d8 --- /dev/null +++ b/app/services/domain_heuristic/__init__.py @@ -0,0 +1,3 @@ +from app.services.domain_heuristic.check import check_domain_heuristic + +__all__ = ["check_domain_heuristic"] diff --git a/app/services/domain_heuristic/patterns.py b/app/services/domain_heuristic/patterns.py new file mode 100644 index 0000000..a6d515d --- /dev/null +++ b/app/services/domain_heuristic/patterns.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import ipaddress +from urllib.parse import parse_qs, urlparse + +import tldextract + +from app.core.config import settings +from app.schemas.domain_heuristic import DomainHeuristicSignal + +_SUSPICIOUS_TLDS = frozenset( + { + "zip", + "mov", + "xyz", + "top", + "pw", + "tk", + "ml", + "ga", + "cf", + "gq", + "fit", + "work", + "surf", + "click", + "link", + "loan", + "download", + "stream", + "racing", + "review", + "trade", + "win", + "bid", + "party", + "date", + "faith", + "science", + "cricket", + "accountant", + "men", + } +) + +HOSTING_PLATFORMS = frozenset( + { + "netlify.app", + "vercel.app", + "pages.dev", + "github.io", + "gitlab.io", + "web.app", + "firebaseapp.com", + "amazonaws.com", + "azurewebsites.net", + "herokuapp.com", + "glitch.me", + "replit.app", + "surge.sh", + "render.com", + "fly.dev", + "railway.app", + "cyclic.app", + } +) + +_OPEN_REDIRECT_PARAMS = frozenset( + { + "url", + "redirect", + "redirect_url", + "redirect_uri", + "next", + "return", + "return_url", + "returnurl", + "goto", + "destination", + "target", + "link", + "forward", + "continue", + "redir", + "out", + } +) + + +def _is_ip_address(host: str) -> bool: + # IPv6는 urlparse가 [] 제거하지 않으므로 strip 필요 + host = host.strip("[]") + try: + ipaddress.ip_address(host) + return True + except ValueError: + return False + + +def check_patterns(url: str) -> list[DomainHeuristicSignal]: + signals: list[DomainHeuristicSignal] = [] + parsed = urlparse(url) + host = parsed.hostname or "" + + if _is_ip_address(host): + signals.append(DomainHeuristicSignal.IP_DIRECT) + return signals # IP면 도메인 기반 신호 분석 불필요 + + if parsed.scheme == "http": + signals.append(DomainHeuristicSignal.NO_HTTPS) + + ext = tldextract.extract(url) + + # 서브도메인 레이블 과다 + subdomain_labels = [s for s in ext.subdomain.split(".") if s] + if len(subdomain_labels) >= settings.subdomain_label_threshold: + signals.append(DomainHeuristicSignal.SUBDOMAIN_OVERUSE) + + # Punycode/IDN 호모글리프 + if "xn--" in host: + signals.append(DomainHeuristicSignal.PUNYCODE_IDN) + + # 하이픈 과다 (등록 도메인 레이블 기준) + domain_label = ext.domain + hyphen_count = domain_label.count("-") + if ( + hyphen_count >= settings.hyphen_count_threshold + or len(domain_label) >= settings.domain_label_length_threshold + ): + signals.append(DomainHeuristicSignal.HYPHEN_OVERUSE) + + # 의심 TLD + if ext.suffix in _SUSPICIOUS_TLDS: + signals.append(DomainHeuristicSignal.SUSPICIOUS_TLD) + + # 오픈 리다이렉트 파라미터 + qs = parse_qs(parsed.query, keep_blank_values=True) + if any(k.lower() in _OPEN_REDIRECT_PARAMS for k in qs): + signals.append(DomainHeuristicSignal.OPEN_REDIRECT_PARAM) + + # 합법 호스팅 플랫폼 (domain.suffix 조합으로 검사) + if ext.suffix and ext.subdomain: + domain_suffix = f"{ext.domain}.{ext.suffix}" + if domain_suffix in HOSTING_PLATFORMS or ext.suffix in HOSTING_PLATFORMS: + signals.append(DomainHeuristicSignal.HOSTING_PLATFORM) + + return signals diff --git a/tests/services/domain_heuristic/__init__.py b/tests/services/domain_heuristic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/domain_heuristic/test_patterns.py b/tests/services/domain_heuristic/test_patterns.py new file mode 100644 index 0000000..d09b41e --- /dev/null +++ b/tests/services/domain_heuristic/test_patterns.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.patterns import check_patterns + + +def test_ip_direct_ipv4() -> None: + signals = check_patterns("http://192.168.1.1/login") + assert DomainHeuristicSignal.IP_DIRECT in signals + + +def test_ip_direct_ipv6() -> None: + signals = check_patterns("http://[::1]/login") + assert DomainHeuristicSignal.IP_DIRECT in signals + + +def test_ip_direct_returns_only_ip_signal() -> None: + # IP 감지 시 다른 도메인 신호는 분석하지 않음 + signals = check_patterns("http://192.168.1.1/login?redirect=evil") + assert signals == [DomainHeuristicSignal.IP_DIRECT] + + +def test_no_https() -> None: + signals = check_patterns("http://example.com/") + assert DomainHeuristicSignal.NO_HTTPS in signals + + +def test_https_no_flag() -> None: + signals = check_patterns("https://example.com/") + assert DomainHeuristicSignal.NO_HTTPS not in signals + + +def test_subdomain_overuse() -> None: + signals = check_patterns("https://a.b.c.d.example.com/") + assert DomainHeuristicSignal.SUBDOMAIN_OVERUSE in signals + + +def test_subdomain_normal() -> None: + signals = check_patterns("https://login.naver.com/") + assert DomainHeuristicSignal.SUBDOMAIN_OVERUSE not in signals + + +def test_punycode_idn() -> None: + signals = check_patterns("https://xn--naver-zk5b.com/") + assert DomainHeuristicSignal.PUNYCODE_IDN in signals + + +def test_hyphen_overuse_count() -> None: + signals = check_patterns("https://login-secure-naver-auth.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE in signals + + +def test_hyphen_overuse_length() -> None: + # 20자 이상 레이블 → HYPHEN_OVERUSE 발동 + signals = check_patterns("https://averylongsingledomainlabelname.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE in signals + + +def test_no_hyphen_overuse() -> None: + signals = check_patterns("https://naver.com/") + assert DomainHeuristicSignal.HYPHEN_OVERUSE not in signals + + +def test_suspicious_tld_xyz() -> None: + signals = check_patterns("https://example.xyz/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in signals + + +def test_suspicious_tld_zip() -> None: + signals = check_patterns("https://example.zip/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD in signals + + +def test_normal_tld() -> None: + signals = check_patterns("https://example.com/") + assert DomainHeuristicSignal.SUSPICIOUS_TLD not in signals + + +def test_open_redirect_param() -> None: + signals = check_patterns("https://example.com/login?redirect=http://evil.com") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM in signals + + +def test_open_redirect_param_url() -> None: + signals = check_patterns("https://example.com/go?url=http://evil.com") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM in signals + + +def test_no_open_redirect_param() -> None: + signals = check_patterns("https://example.com/search?q=hello") + assert DomainHeuristicSignal.OPEN_REDIRECT_PARAM not in signals + + +def test_hosting_platform_netlify() -> None: + signals = check_patterns("https://myapp.netlify.app/") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + + +def test_hosting_platform_github_io() -> None: + signals = check_patterns("https://user.github.io/repo") + assert DomainHeuristicSignal.HOSTING_PLATFORM in signals + + +def test_normal_domain_no_hosting_flag() -> None: + signals = check_patterns("https://naver.com/") + assert DomainHeuristicSignal.HOSTING_PLATFORM not in signals + + +def test_hosting_platform_root_not_flagged() -> None: + # 플랫폼 자체 홈페이지는 HOSTING_PLATFORM 미발동 + signals = check_patterns("https://netlify.com/") + assert DomainHeuristicSignal.HOSTING_PLATFORM not in signals From 439ec24a4cc71672fba981b70d90db8b46087f6b Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:04:48 +0900 Subject: [PATCH 065/173] =?UTF-8?q?[Feat]=20DGA=20=EC=9D=98=EC=8B=AC=20?= =?UTF-8?q?=EB=8F=84=EB=A9=94=EC=9D=B8=20=EA=B2=80=EC=82=AC=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20(Shannon=20=EC=97=94=ED=8A=B8=EB=A1=9C=ED=94=BC?= =?UTF-8?q?=C2=B7=EC=9E=90=EC=9D=8C=20=EB=B9=84=EC=9C=A8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/domain_heuristic/dga.py | 49 +++++++++++ tests/services/domain_heuristic/test_dga.py | 97 +++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 app/services/domain_heuristic/dga.py create mode 100644 tests/services/domain_heuristic/test_dga.py diff --git a/app/services/domain_heuristic/dga.py b/app/services/domain_heuristic/dga.py new file mode 100644 index 0000000..2d09f1c --- /dev/null +++ b/app/services/domain_heuristic/dga.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import math + +import tldextract + +from app.core.config import settings +from app.schemas.domain_heuristic import DomainHeuristicSignal + +_CONSONANTS = frozenset("bcdfghjklmnpqrstvwxyz") + + +_MIN_DGA_LABEL_LEN = 8 # netflix(7), spotify(7) 같은 짧은 영어 브랜드 오탐 방지 + + +def _shannon_entropy(s: str) -> float: + if not s: + return 0.0 + freq: dict[str, int] = {} + for c in s: + freq[c] = freq.get(c, 0) + 1 + n = len(s) + total = sum((count / n) * math.log2(count / n) for count in freq.values()) + return -total + + +def _consonant_ratio(s: str) -> float: + letters = [c for c in s.lower() if c.isalpha()] + if not letters: + return 0.0 + return sum(1 for c in letters if c in _CONSONANTS) / len(letters) + + +def check_dga(url: str) -> DomainHeuristicSignal | None: + ext = tldextract.extract(url) + label = ext.domain # 등록 도메인 레이블 + if not label or len(label) < _MIN_DGA_LABEL_LEN: + # 짧은 레이블은 엔트로피/자음 비율 통계가 불안정 → 스킵 + return None + + entropy = _shannon_entropy(label) + consonant_ratio = _consonant_ratio(label) + + if ( + entropy >= settings.dga_entropy_threshold + or consonant_ratio >= settings.dga_consonant_ratio_threshold + ): + return DomainHeuristicSignal.DGA_LIKE + return None diff --git a/tests/services/domain_heuristic/test_dga.py b/tests/services/domain_heuristic/test_dga.py new file mode 100644 index 0000000..8d78918 --- /dev/null +++ b/tests/services/domain_heuristic/test_dga.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.dga import ( + _consonant_ratio, + _shannon_entropy, + check_dga, +) + + +def test_shannon_entropy_uniform() -> None: + # 모든 문자가 동일하면 엔트로피 0 + assert _shannon_entropy("aaaa") == pytest.approx(0.0) + + +def test_shannon_entropy_high() -> None: + # 무작위 문자열은 엔트로피 높음 + entropy = _shannon_entropy("xkzqjvpm") + assert entropy >= 3.0 + + +def test_shannon_entropy_empty() -> None: + assert _shannon_entropy("") == 0.0 + + +def test_consonant_ratio_all_consonants() -> None: + ratio = _consonant_ratio("bxkrptmn") + assert ratio == pytest.approx(1.0) + + +def test_consonant_ratio_all_vowels() -> None: + ratio = _consonant_ratio("aeiou") + assert ratio == pytest.approx(0.0) + + +def test_consonant_ratio_empty() -> None: + assert _consonant_ratio("") == 0.0 + + +def test_dga_high_entropy_flagged() -> None: + # "xkzqjvpmbn" — 높은 엔트로피 + 자음 비율 + signal = check_dga("https://xkzqjvpmbn.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_normal_domain_not_flagged() -> None: + signal = check_dga("https://naver.com/") + assert signal is None + + +def test_dga_google_not_flagged() -> None: + signal = check_dga("https://google.com/") + assert signal is None + + +def test_dga_high_consonant_ratio() -> None: + # "bxkrptmnsdf" — 자음 비율 > 0.6 + signal = check_dga("https://bxkrptmnsdf.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_ip_url_not_flagged() -> None: + # IP URL — 도메인 레이블 없음 + signal = check_dga("http://192.168.1.1/") + assert signal is None + + +def test_dga_uses_registered_domain_label() -> None: + # 서브도메인 무시, 등록 도메인 레이블만 분석 + # "naver"는 정상 엔트로피 → 미발동 + signal = check_dga("https://login.naver.com/") + assert signal is None + + +def test_dga_entropy_only_triggers() -> None: + # 자음 비율 < 0.8이지만 Shannon 엔트로피 >= 3.5 → DGA_LIKE 발동 + # "aeioubcdfghj": 12개 고유 문자, 엔트로피=3.585, 자음 비율=0.583 + signal = check_dga("https://aeioubcdfghj.com/") + assert signal == DomainHeuristicSignal.DGA_LIKE + + +def test_dga_short_label_skipped() -> None: + # 레이블 < 8자는 통계 신뢰도 낮아 스킵 — netflix/spotify 오탐 방지 + assert check_dga("https://netflix.com/") is None + assert check_dga("https://spotify.com/") is None + assert check_dga("https://samsung.com/") is None + + +def test_dga_consonant_ratio_below_new_threshold() -> None: + # "snapchat" 8자, 자음 비율 6/8=0.75 < 0.8 → 미발동 + assert check_dga("https://snapchat.com/") is None + + +def test_dga_long_dga_like_still_flagged() -> None: + # 8자 이상 + 자음 비율 >= 0.8 → 여전히 발동 + assert check_dga("https://bxkrptmnsdf.com/") == DomainHeuristicSignal.DGA_LIKE From 0bb77ee67237f2b8ec9d6dee2fecf2a8cf834f86 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:05:22 +0900 Subject: [PATCH 066/173] =?UTF-8?q?[Feat]=20=ED=83=80=EC=9D=B4=ED=8F=AC?= =?UTF-8?q?=EC=8A=A4=EC=BF=BC=ED=8C=85=20=EA=B2=80=EC=82=AC=20=EB=B0=8F=20?= =?UTF-8?q?=EB=B8=8C=EB=9E=9C=EB=93=9C=20=ED=99=94=EC=9D=B4=ED=8A=B8?= =?UTF-8?q?=EB=A6=AC=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/domain_heuristic/brands.txt | 603 ++++++++++++++++++ .../domain_heuristic/typosquatting.py | 90 +++ .../domain_heuristic/test_typosquatting.py | 114 ++++ 3 files changed, 807 insertions(+) create mode 100644 app/services/domain_heuristic/brands.txt create mode 100644 app/services/domain_heuristic/typosquatting.py create mode 100644 tests/services/domain_heuristic/test_typosquatting.py diff --git a/app/services/domain_heuristic/brands.txt b/app/services/domain_heuristic/brands.txt new file mode 100644 index 0000000..4cc00b9 --- /dev/null +++ b/app/services/domain_heuristic/brands.txt @@ -0,0 +1,603 @@ +# brands.txt — 피싱/타이포스쿼팅 탐지용 브랜드 도메인 목록 +# 형식: 한 줄에 도메인 하나, # 주석 허용 + +# ───────────────────────────────────────── +# 국내 포털 / SNS +# ───────────────────────────────────────── +naver.com +navercloud.com +navercorp.com +daum.net +daum.com +nate.com +kakao.com +kakaotalk.com +kakaocorp.com +kakaoenterprise.com +band.us +band.naver.com +cyworld.com +me2day.net +blog.naver.com +cafe.naver.com +map.naver.com +shopping.naver.com +news.naver.com +finance.naver.com +webtoon.naver.com + +# ───────────────────────────────────────── +# 국내 금융 — 은행 +# ───────────────────────────────────────── +kbstar.com +kbbank.com +kb.co.kr +shinhan.com +shinhanbank.com +wooribank.com +woori.com +ibk.co.kr +ibkbank.com +hanabank.com +hana.com +bnk.co.kr +nonghyup.com +nh.com +nhbank.com +kebhana.com +saemaeulgeumgo.or.kr +busan.bank +daegubank.com +gwangjubank.com +jeonbukbank.com +jejubank.com +scbank.co.kr +citybank.co.kr +citi.co.kr +kakaobank.com +kbank.co.kr +tossbank.com +suhyup.com +nfcf.co.kr + +# ───────────────────────────────────────── +# 국내 금융 — 카드 / 간편결제 / 핀테크 +# ───────────────────────────────────────── +kakaopay.com +toss.im +tossbank.com +payco.com +samsungcard.com +lottecard.com +hyundaicard.com +bccard.com +shinhancard.com +wooricard.com +kbcard.com +nonghyupcard.com +hanacard.com +lottefinance.com +ssgincome.com +zeropay.or.kr +naver.pay +ssgpay.com +lpay.com +chai.finance +ypay.co.kr + +# ───────────────────────────────────────── +# 국내 금융 — 증권 / 자산운용 / 보험 +# ───────────────────────────────────────── +kiwoom.com +miraeasset.com +samsungfund.com +hdsecurities.com +nhqv.com +daishin.com +hi-investment.com +eugenefn.com +sk.securities +dbfinancial.com +koreinvestment.com +meritz.co.kr +samsung.life +hanwhalife.com +kyobolife.com +linakorea.com +axa.co.kr +db.insurance +hyundai.insurance +lotte.insurance + +# ───────────────────────────────────────── +# 국내 증권 / 암호화폐 거래소 +# ───────────────────────────────────────── +upbit.com +bithumb.com +korbit.co.kr +coinone.co.kr +gopax.co.kr +huobi.co.kr +coinplug.com +bitsum.com +probit.com +flata.exchange +coredax.com + +# ───────────────────────────────────────── +# 국내 이커머스 +# ───────────────────────────────────────── +coupang.com +coupangplay.com +gmarket.co.kr +auction.co.kr +11st.co.kr +interpark.com +tmon.co.kr +wemakeprice.com +ssg.com +lotteon.com +lotteshopping.com +kurly.com +marketkurly.com +musinsa.com +zigzag.kr +brandi.co.kr +ably.kr +29cm.co.kr +hm.com +zara.com +shein.com +aboutyou.com +fashionplate.co.kr +nsmall.com +hyundaishopping.com +hmall.com +emart.com +emarttraders.com +lottemart.com +homeplus.co.kr +costco.co.kr +iherb.com +elevenstreet.co.kr + +# ───────────────────────────────────────── +# 국내 배달 / O2O / 라이프스타일 +# ───────────────────────────────────────── +baemin.com +yogiyo.co.kr +coupangeats.com +shuttledelivery.com +naver.place +kakaomap.com +kakaonaviapp.com +kakaomobility.com +socar.kr +greencar.com +lottaxi.com +kakao.taxi +ddangyo.com +yanolja.com +goodchoice.kr +airbnb.co.kr +syranos.com +myrealtrip.com +naver.hotel + +# ───────────────────────────────────────── +# 국내 통신 +# ───────────────────────────────────────── +kt.com +olleh.com +tworld.co.kr +skt.com +sktelecom.com +lguplus.com +uplus.co.kr +lg.com +skbroadband.com +ktcs.co.kr +ktwiz.co.kr +mvno.skt.com +mvno.kt.com + +# ───────────────────────────────────────── +# 국내 공공 / 정부 +# ───────────────────────────────────────── +gov.kr +nhis.or.kr +nps.or.kr +hometax.go.kr +nts.go.kr +fss.or.kr +kisa.or.kr +kcb.co.kr +nice.co.kr +mois.go.kr +mohw.go.kr +moef.go.kr +moj.go.kr +cdc.go.kr +police.go.kr +customs.go.kr +epts.go.kr +eps.go.kr +work.go.kr +koreajobworld.com +epeople.go.kr +wetax.go.kr +irs.go.kr +judicial.go.kr +scourt.go.kr +patent.go.kr +kipo.go.kr +mfds.go.kr +mida.re.kr +kofia.or.kr +krx.co.kr + +# ───────────────────────────────────────── +# 국내 미디어 / 콘텐츠 +# ───────────────────────────────────────── +webtoon.com +kakaopage.com +ridibooks.com +yes24.com +watcha.com +wavve.com +tving.com +melon.com +bugs.co.kr +flocloud.kr +genie.co.kr +vibe.naver.com +chosun.com +joongang.co.kr +donga.com +yonhapnews.co.kr +mbc.co.kr +kbs.co.kr +sbs.co.kr +jtbc.co.kr +tvn.com +ocn.com +ytn.co.kr +khan.co.kr +hani.co.kr +ohmynews.com +pressian.com +mediaus.co.kr +newsis.com +newspim.com +edaily.co.kr +mt.co.kr +ajunews.com +yna.co.kr + +# ───────────────────────────────────────── +# 국내 IT / 스타트업 +# ───────────────────────────────────────── +krafton.com +ncsoft.com +nexon.com +netmarble.com +smilegate.com +com2us.com +kakao.games +kakao.corp +line.me +naverwebtoon.com +ncloud.com +gabia.com +cafe24.com +hosting.kr +iwinv.com +whois.co.kr +godaddy.co.kr + +# ───────────────────────────────────────── +# 글로벌 빅테크 +# ───────────────────────────────────────── +google.com +gmail.com +youtube.com +google.co.kr +google.co.jp +google.co.uk +google.de +google.fr +google.com.au +googleads.com +googleanalytics.com +googlepay.com +apple.com +icloud.com +appleid.apple.com +itunes.com +appstore.com +microsoft.com +outlook.com +hotmail.com +live.com +office.com +office365.com +teams.microsoft.com +azure.com +windowsazure.com +microsoftonline.com +onedrive.com +skype.com +xbox.com +amazon.com +aws.amazon.com +amazonpay.com +primevideo.com +audible.com +twitch.tv +meta.com +facebook.com +instagram.com +whatsapp.com +messenger.com +threads.net +twitter.com +x.com +linkedin.com +pinterest.com +snapchat.com +tiktok.com +reddit.com +tumblr.com +quora.com +medium.com +substack.com + +# ───────────────────────────────────────── +# 글로벌 금융 +# ───────────────────────────────────────── +paypal.com +stripe.com +wise.com +revolut.com +mastercard.com +visa.com +americanexpress.com +amex.com +citibank.com +bankofamerica.com +chase.com +wellsfargo.com +hsbc.com +hsbc.co.uk +barclays.com +lloydsbank.com +natwest.com +santander.com +ubs.com +credit-suisse.com +deutschebank.com +bnpparibas.com +societegenale.com +inggroup.com +ing.com +commerzbank.com +rabobank.com +abnamro.com +coinbase.com +binance.com +kraken.com +bitfinex.com +gemini.com +ftx.com +crypto.com +bybit.com +okx.com +kucoin.com + +# ───────────────────────────────────────── +# 글로벌 이커머스 / 소매 +# ───────────────────────────────────────── +ebay.com +etsy.com +shopify.com +aliexpress.com +alibaba.com +walmart.com +target.com +bestbuy.com +homedepot.com +costco.com +wayfair.com +overstock.com +newegg.com +rakuten.com +lazada.com +shopee.com +tokopedia.com +bukalapak.com +flipkart.com +jd.com +taobao.com +tmall.com +pinduoduo.com + +# ───────────────────────────────────────── +# 글로벌 클라우드 / DevTools / 협업 +# ───────────────────────────────────────── +github.com +gitlab.com +bitbucket.org +stackoverflow.com +docker.com +dockerhub.com +cloudflare.com +digitalocean.com +notion.so +atlassian.com +jira.atlassian.com +confluence.atlassian.com +slack.com +discord.com +zoom.us +figma.com +miro.com +trello.com +asana.com +monday.com +airtable.com +clickup.com +linear.app +vercel.com +netlify.com +heroku.com +render.com +railway.app +supabase.com +firebase.google.com +amplify.aws +terraform.io +ansible.com +kubernetes.io +helm.sh +grafana.com +datadog.com +newrelic.com +splunk.com +elastic.co +kibana.com +prometheus.io +circleci.com +travisci.com +jenkins.io +githubactions.com +argocd.io + +# ───────────────────────────────────────── +# 글로벌 보안 +# ───────────────────────────────────────── +auth0.com +okta.com +lastpass.com +1password.com +bitwarden.com +norton.com +mcafee.com +kaspersky.com +malwarebytes.com +crowdstrike.com +paloaltonetworks.com +sentinelone.com +darktrace.com +fortinet.com +checkpoint.com +sophos.com +trendmicro.com +avast.com +avg.com +eset.com +bitdefender.com +cylance.com +qualys.com +rapid7.com +tenable.com + +# ───────────────────────────────────────── +# 글로벌 여행 / 숙박 +# ───────────────────────────────────────── +booking.com +airbnb.com +expedia.com +tripadvisor.com +hotels.com +agoda.com +kayak.com +skyscanner.com +momondo.com +trivago.com +vrbo.com +marriott.com +hilton.com +hyatt.com +ihg.com +bestwestern.com +radissonhotels.com +accor.com + +# ───────────────────────────────────────── +# 글로벌 게임 +# ───────────────────────────────────────── +steam.com +steampowered.com +epicgames.com +roblox.com +ea.com +origin.com +ubisoft.com +blizzard.com +battlenet.com +playstation.com +xbox.com +nintendo.com +itch.io +gog.com +humblebundle.com +razer.com + +# ───────────────────────────────────────── +# 글로벌 미디어 / 스트리밍 / SaaS +# ───────────────────────────────────────── +netflix.com +spotify.com +dropbox.com +wordpress.com +wix.com +godaddy.com +twilio.com +sendgrid.com +salesforce.com +zendesk.com +hubspot.com +mailchimp.com +constantcontact.com +marketo.com +pardot.com +servicenow.com +workday.com +sap.com +oracle.com +adobe.com +canva.com +squarespace.com +weebly.com +shopify.ca +shopify.co.uk +bigcommerce.com +magento.com +prestashop.com +opencart.com +woocommerce.com +hulu.com +disneyplus.com +hbomax.com +peacocktv.com +paramountplus.com +appletv.com +crunchyroll.com +funimation.com +vimeo.com +dailymotion.com +twitch.com +youtube.tv +soundcloud.com +tidal.com +deezer.com +pandora.com +iheartradio.com +audible.com +kindle.com +scribd.com +coursera.com +udemy.com +edx.org +khanacademy.org +duolingo.com diff --git a/app/services/domain_heuristic/typosquatting.py b/app/services/domain_heuristic/typosquatting.py new file mode 100644 index 0000000..585e00a --- /dev/null +++ b/app/services/domain_heuristic/typosquatting.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from pathlib import Path + +import tldextract + +from app.core.logging import get_logger +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic.patterns import HOSTING_PLATFORMS + +logger = get_logger(__name__) + +_BRANDS_FILE = Path(__file__).parent / "brands.txt" + + +def _load_brands() -> list[tuple[str, str]]: + """brands.txt를 읽어 (domain_label, tld) 튜플 리스트로 반환. 모듈 로드 시 1회 실행.""" + if not _BRANDS_FILE.exists(): + # silent fail은 typosquatting 검사 영구 무력화로 이어져 false negative 폭증 — 명시적 로깅 + logger.error("typosquatting.brands_file_missing", path=str(_BRANDS_FILE)) + return [] + seen: set[tuple[str, str]] = set() + brands: list[tuple[str, str]] = [] + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + ext = tldextract.extract(f"https://{line}") + if ext.domain and ext.suffix: + key = (ext.domain, ext.suffix) + if key not in seen: + seen.add(key) + brands.append(key) + if not brands: + logger.error("typosquatting.brands_empty", path=str(_BRANDS_FILE)) + return brands + + +_BRANDS: list[tuple[str, str]] = _load_brands() + + +def _levenshtein(a: str, b: str) -> int: + """레벤슈타인 거리 — 외부 라이브러리 없이 DP로 구현.""" + m, n = len(a), len(b) + dp = list(range(n + 1)) + for i in range(1, m + 1): + prev = dp[0] + dp[0] = i + for j in range(1, n + 1): + temp = dp[j] + if a[i - 1] == b[j - 1]: + dp[j] = prev + else: + dp[j] = 1 + min(prev, dp[j], dp[j - 1]) + prev = temp + return dp[n] + + +_MIN_FUZZY_LEN = 5 # 4자 이하 브랜드는 편집거리 1~2 오탐이 심해 완전일치만 인정 + + +def check_typosquatting(url: str) -> DomainHeuristicSignal | None: + ext = tldextract.extract(url) + if not ext.top_domain_under_public_suffix: + return None + + target_domain = ext.domain + target_suffix = ext.suffix + + # 호스팅 플랫폼 자체 도메인(netlify.app, vercel.app 등)은 brand 변종(netlify.com)과 + # label 동일 + suffix 상이로 TYPO_DOMAIN 오탐되므로 조기 컷아웃 + if ext.top_domain_under_public_suffix in HOSTING_PLATFORMS: + return None + + is_typo = False + for brand_domain, brand_suffix in _BRANDS: + if target_domain == brand_domain and target_suffix == brand_suffix: + return None # 완전 일치 → 정상 도메인 + + if abs(len(target_domain) - len(brand_domain)) > 2: + continue # 길이 차이가 2 초과면 거리가 반드시 2 초과 → 스킵 + + distance = _levenshtein(target_domain, brand_domain) + if distance == 0 and target_suffix != brand_suffix: + is_typo = True + elif 1 <= distance <= 2 and min(len(target_domain), len(brand_domain)) >= _MIN_FUZZY_LEN: + # "ing" vs "bing", "kb" vs "kbx" 같은 단어가 정상 도메인을 오탐하는 문제 차단 + is_typo = True + + return DomainHeuristicSignal.TYPO_DOMAIN if is_typo else None diff --git a/tests/services/domain_heuristic/test_typosquatting.py b/tests/services/domain_heuristic/test_typosquatting.py new file mode 100644 index 0000000..7bebf8f --- /dev/null +++ b/tests/services/domain_heuristic/test_typosquatting.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicSignal +from app.services.domain_heuristic import typosquatting +from app.services.domain_heuristic.typosquatting import _levenshtein, check_typosquatting + + +def test_levenshtein_same(): + assert _levenshtein("naver", "naver") == 0 + + +def test_levenshtein_one_insert(): + assert _levenshtein("naverr", "naver") == 1 + + +def test_levenshtein_one_substitute(): + assert _levenshtein("navcr", "naver") == 1 + + +def test_levenshtein_two(): + assert _levenshtein("naverr", "naver") == 1 + assert _levenshtein("naaver", "naver") == 1 + + +def test_typo_distance_1(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverr.com/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_typo_distance_2(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverro.com/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_exact_match_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naver.com/") + assert signal is None + + +def test_distance_3_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naverrrr.com/") + assert signal is None + + +def test_same_label_different_tld(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("https://naver.net/") + assert signal == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_no_registered_domain(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("naver", "com")]) + signal = check_typosquatting("http://localhost/") + assert signal is None + + +def test_empty_brands(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", []) + signal = check_typosquatting("https://naverr.com/") + assert signal is None + + +def test_multi_tld_brand_com_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # net 먼저 등록된 브랜드라도 com 버전은 정상 도메인으로 처리 + monkeypatch.setattr(typosquatting, "_BRANDS", [("daum", "net"), ("daum", "com")]) + assert check_typosquatting("https://daum.com/") is None + + +def test_multi_tld_brand_tv_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # tv 먼저 등록된 브랜드라도 com 버전은 정상 도메인으로 처리 + monkeypatch.setattr(typosquatting, "_BRANDS", [("twitch", "tv"), ("twitch", "com")]) + assert check_typosquatting("https://twitch.com/") is None + + +def test_short_brand_no_fuzzy_match(monkeypatch: pytest.MonkeyPatch) -> None: + # 3자 브랜드 "ing" 기준 bing.com(4자)은 편집거리 1이지만 min(3,4)<5 → 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("ing", "com")]) + assert check_typosquatting("https://bing.com/") is None + + +def test_short_brand_cnn_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # 3자 브랜드 "ocn"과 cnn은 편집거리 2지만 짧은 브랜드라 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("ocn", "com")]) + assert check_typosquatting("https://cnn.com/") is None + + +def test_two_char_brand_no_fuzzy(monkeypatch: pytest.MonkeyPatch) -> None: + # 2자 브랜드 "kb"는 완전 일치 외 모두 스킵 → kbs.com 오탐 방지 + monkeypatch.setattr(typosquatting, "_BRANDS", [("kb", "co.kr")]) + assert check_typosquatting("https://kbs.com/") is None + + +def test_short_brand_exact_label_diff_suffix_still_flagged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 정확히 같은 label이면 suffix 달라도 여전히 typosquat 취급 (distance 0은 길이 무관) + monkeypatch.setattr(typosquatting, "_BRANDS", [("kb", "co.kr")]) + assert check_typosquatting("https://kb.xyz/") == DomainHeuristicSignal.TYPO_DOMAIN + + +def test_hosting_platform_root_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + # netlify.app 루트가 brand netlify.com과 label 일치 + suffix 상이로 오탐되던 케이스 + monkeypatch.setattr(typosquatting, "_BRANDS", [("netlify", "com")]) + assert check_typosquatting("https://netlify.app/") is None + + +def test_hosting_platform_root_vercel_not_flagged(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(typosquatting, "_BRANDS", [("vercel", "com")]) + assert check_typosquatting("https://vercel.app/") is None From fa2caeca43532d6b64a66476e35654a1dc26ae01 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:05:52 +0900 Subject: [PATCH 067/173] =?UTF-8?q?[Feat]=20RDAP=20=EC=A1=B0=ED=9A=8C=20?= =?UTF-8?q?=EB=AA=A8=EB=93=88=20=EC=B6=94=EA=B0=80=20(client=20=EC=8B=B1?= =?UTF-8?q?=EA=B8=80=ED=84=B4=C2=B724h=20TTL=20=EC=BA=90=EC=8B=9C=C2=B7in-?= =?UTF-8?q?flight=20dedup)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/domain_heuristic/rdap.py | 174 +++++++++++++ tests/services/domain_heuristic/test_rdap.py | 242 +++++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 app/services/domain_heuristic/rdap.py create mode 100644 tests/services/domain_heuristic/test_rdap.py diff --git a/app/services/domain_heuristic/rdap.py b/app/services/domain_heuristic/rdap.py new file mode 100644 index 0000000..209263c --- /dev/null +++ b/app/services/domain_heuristic/rdap.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import asyncio +import time +from datetime import UTC, datetime + +import httpx +import tldextract + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.domain_heuristic import RdapInfo + +logger = get_logger(__name__) + +# domain → (RdapInfo | None, expire_at: monotonic timestamp) +_cache: dict[str, tuple[RdapInfo | None, float]] = {} + +# 동시 요청 합치기용 — 같은 도메인 요청이 겹치면 하나의 Future만 실제로 fetch +_inflight: dict[str, asyncio.Future[tuple[RdapInfo | None, str | None]]] = {} + +# 커넥션/TLS 재사용을 위해 모듈 레벨에서 싱글턴 사용 +_client: httpx.AsyncClient | None = None + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None: + # RFC 7480 SHOULD: authoritative RDAP 서버로 redirect될 때 일부 구현은 + # Accept 없으면 406 또는 HTML을 반환 → 기본 헤더로 명시 + _client = httpx.AsyncClient( + timeout=settings.rdap_timeout_seconds, + headers={"Accept": "application/rdap+json"}, + ) + return _client + + +async def aclose_client() -> None: + """앱 셧다운 훅에서 호출. 테스트에서 client 상태 초기화에도 사용.""" + global _client + if _client is not None: + await _client.aclose() + _client = None + + +def _extract_registrar(data: dict) -> str | None: # type: ignore[type-arg] + for entity in data.get("entities", []): + roles = entity.get("roles", []) + if "registrar" in roles: + vcard = entity.get("vcardArray", []) + if isinstance(vcard, list) and len(vcard) > 1: + for entry in vcard[1]: + if isinstance(entry, list) and entry and entry[0] == "fn": + return str(entry[3]) + return None + + +def _parse_datetime(s: str | None) -> datetime | None: + if not s: + return None + try: + # Python 3.11+의 fromisoformat은 Z suffix를 직접 파싱 + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + except ValueError: + return None + + +def _parse_rdap_response(domain: str, data: dict) -> RdapInfo: # type: ignore[type-arg] + events = {e["eventAction"]: e.get("eventDate") for e in data.get("events", [])} + created_date = _parse_datetime(events.get("registration")) + expiry_date = _parse_datetime(events.get("expiration")) + + now = datetime.now(tz=UTC) + domain_age_days: int | None = None + is_new_domain = False + if created_date: + domain_age_days = (now - created_date).days + is_new_domain = domain_age_days < settings.rdap_new_domain_threshold_days + + return RdapInfo( + domain=domain, + registrar=_extract_registrar(data), + created_date=created_date, + expiry_date=expiry_date, + domain_age_days=domain_age_days, + is_new_domain=is_new_domain, + ) + + +def _encode_for_url(domain: str) -> str | None: + """IDN 도메인을 punycode로 변환. ASCII 도메인은 그대로 통과.""" + try: + return domain.encode("idna").decode("ascii") + except UnicodeError: + return None + + +async def _fetch_rdap(domain: str) -> tuple[RdapInfo | None, str | None]: + encoded = _encode_for_url(domain) + if encoded is None: + return None, "invalid_domain" + + rdap_url = f"{settings.rdap_bootstrap_url}{encoded}" + try: + client = _get_client() + resp = await client.get(rdap_url, follow_redirects=True) + resp.raise_for_status() + data = resp.json() + except httpx.TimeoutException: + return None, "timeout" + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return None, "not_found" + return None, "http_error" + except Exception as exc: + logger.warning( + "rdap.unexpected_error", + domain=domain, + error=str(exc), + error_type=type(exc).__name__, + ) + return None, "unexpected" + + try: + result = _parse_rdap_response(domain, data) + except Exception as exc: + logger.warning( + "rdap.parse_error", + domain=domain, + error=str(exc), + error_type=type(exc).__name__, + ) + return None, "parse_error" + + _cache[domain] = (result, time.monotonic() + settings.rdap_cache_ttl_seconds) + return result, None + + +async def lookup_rdap(url: str) -> tuple[RdapInfo | None, str | None]: + """(RdapInfo | None, error_code | None) 반환. 에러 시 파이프라인 중단하지 않음.""" + ext = tldextract.extract(url) + domain = ext.top_domain_under_public_suffix + if not domain: + return None, "no_domain" + + now = time.monotonic() + cached = _cache.get(domain) + if cached is not None: + info, expire_at = cached + if now < expire_at: + return info, None + # 만료 엔트리는 즉시 제거 — 무작위 도메인 트래픽으로 dict 무한 성장 방지 + _cache.pop(domain, None) + + # 같은 도메인 in-flight 요청이 있으면 결과 공유 — RDAP 서버 부하 방지 + existing = _inflight.get(domain) + if existing is not None: + return await existing + + loop = asyncio.get_running_loop() + fut: asyncio.Future[tuple[RdapInfo | None, str | None]] = loop.create_future() + _inflight[domain] = fut + try: + result = await _fetch_rdap(domain) + fut.set_result(result) + return result + except BaseException as exc: + fut.set_exception(exc) + raise + finally: + _inflight.pop(domain, None) diff --git a/tests/services/domain_heuristic/test_rdap.py b/tests/services/domain_heuristic/test_rdap.py new file mode 100644 index 0000000..44aacdf --- /dev/null +++ b/tests/services/domain_heuristic/test_rdap.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from app.schemas.domain_heuristic import RdapInfo +from app.services.domain_heuristic import rdap as rdap_module +from app.services.domain_heuristic.rdap import _parse_datetime, _parse_rdap_response, lookup_rdap + + +@pytest.fixture(autouse=True) +def _reset_rdap_state(monkeypatch: pytest.MonkeyPatch): + """각 테스트에서 cache/inflight/client 싱글턴 상태 격리.""" + rdap_module._cache.clear() + rdap_module._inflight.clear() + monkeypatch.setattr(rdap_module, "_client", None) + + +def _install_client(mock_client: AsyncMock) -> None: + rdap_module._client = mock_client + + +def test_parse_datetime_valid(): + result = _parse_datetime("2026-01-01T00:00:00Z") + assert result is not None + assert result.tzinfo == UTC + assert result.year == 2026 + + +def test_parse_datetime_none(): + assert _parse_datetime(None) is None + + +def test_parse_datetime_invalid(): + assert _parse_datetime("not-a-date") is None + + +def test_parse_rdap_new_domain(): + # 시간 의존을 끊기 위해 "지금-10일" 동적 생성 — settings.rdap_new_domain_threshold_days(30) 미만 + recent = (datetime.now(tz=UTC) - timedelta(days=10)).strftime("%Y-%m-%dT%H:%M:%SZ") + expiry = (datetime.now(tz=UTC) + timedelta(days=355)).strftime("%Y-%m-%dT%H:%M:%SZ") + data = { + "events": [ + {"eventAction": "registration", "eventDate": recent}, + {"eventAction": "expiration", "eventDate": expiry}, + ], + "entities": [ + { + "roles": ["registrar"], + "vcardArray": ["vcard", [["fn", {}, "text", "NameCheap, Inc."]]], + } + ], + } + result = _parse_rdap_response("example.com", data) + assert result.domain == "example.com" + assert result.registrar == "NameCheap, Inc." + assert result.is_new_domain is True + assert result.domain_age_days is not None and result.domain_age_days < 30 + + +def test_parse_rdap_old_domain(): + data = { + "events": [ + {"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}, + ], + "entities": [], + } + result = _parse_rdap_response("old.com", data) + assert result.is_new_domain is False + assert result.domain_age_days is not None and result.domain_age_days > 30 + + +@pytest.mark.asyncio +async def test_lookup_rdap_success(): + mock_response = MagicMock() + mock_response.json.return_value = { + "events": [{"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}], + "entities": [], + } + mock_response.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert error is None + assert result is not None + assert result.domain == "example.com" + + +@pytest.mark.asyncio +async def test_lookup_rdap_cache_hit(): + import time + + fake_info = RdapInfo( + domain="cached.com", + registrar="TestReg", + created_date=None, + expiry_date=None, + domain_age_days=None, + is_new_domain=False, + ) + rdap_module._cache["cached.com"] = (fake_info, time.monotonic() + 3600) + + # client 미설치 — 캐시 히트면 호출 자체가 없어야 함 + result, error = await lookup_rdap("https://cached.com/path") + + assert result == fake_info + assert error is None + + +@pytest.mark.asyncio +async def test_lookup_rdap_timeout(): + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=httpx.TimeoutException("timeout")) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "timeout" + + +@pytest.mark.asyncio +async def test_lookup_rdap_not_found(): + mock_response = MagicMock(status_code=404) + http_error = httpx.HTTPStatusError("not found", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://unknowndomain99999.com/") + + assert result is None + assert error == "not_found" + + +@pytest.mark.asyncio +async def test_lookup_rdap_http_error(): + mock_response = MagicMock(status_code=500) + http_error = httpx.HTTPStatusError("server error", request=MagicMock(), response=mock_response) + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=http_error) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "http_error" + + +@pytest.mark.asyncio +async def test_lookup_rdap_parse_error(): + mock_response = MagicMock() + mock_response.json.return_value = "not a dict" + mock_response.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + _install_client(mock_client) + + result, error = await lookup_rdap("https://example.com/") + + assert result is None + assert error == "parse_error" + + +@pytest.mark.asyncio +async def test_lookup_rdap_dedup_inflight(): + """동시 요청이 겹치면 실제 fetch는 1회만 수행돼야 한다.""" + import asyncio + + call_count = 0 + request_started = asyncio.Event() + release = asyncio.Event() + + async def slow_get(url: str, **_: object) -> MagicMock: + nonlocal call_count + call_count += 1 + request_started.set() + await release.wait() + resp = MagicMock() + resp.json.return_value = { + "events": [{"eventAction": "registration", "eventDate": "2020-01-01T00:00:00Z"}], + "entities": [], + } + resp.raise_for_status = MagicMock() + return resp + + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=slow_get) + _install_client(mock_client) + + task1 = asyncio.create_task(lookup_rdap("https://example.com/")) + task2 = asyncio.create_task(lookup_rdap("https://example.com/")) + task3 = asyncio.create_task(lookup_rdap("https://example.com/")) + await request_started.wait() + release.set() + + results = await asyncio.gather(task1, task2, task3) + + assert call_count == 1 + for info, error in results: + assert error is None + assert info is not None and info.domain == "example.com" + + +@pytest.mark.asyncio +async def test_lookup_rdap_idn_punycode(): + """IDN 도메인은 URL 빌드 시 punycode로 변환돼야 한다.""" + captured_urls: list[str] = [] + + async def capture_get(url: str, **_: object) -> MagicMock: + captured_urls.append(url) + resp = MagicMock() + resp.json.return_value = {"events": [], "entities": []} + resp.raise_for_status = MagicMock() + return resp + + mock_client = AsyncMock() + mock_client.get = AsyncMock(side_effect=capture_get) + _install_client(mock_client) + + await lookup_rdap("https://xn--3e0b707e.com/") # 한국.com + assert captured_urls + assert "xn--3e0b707e.com" in captured_urls[0] + + +@pytest.mark.asyncio +async def test_lookup_rdap_invalid_domain(): + """빈 레이블/비정상 도메인은 invalid_domain 반환.""" + mock_client = AsyncMock() + _install_client(mock_client) + + # tldextract가 도메인으로 받지 않는 형태 — ASCII지만 내부 regex 확인 + # 실제 인코딩 실패 케이스: null byte 포함 + result, error = await lookup_rdap("https://..com/") + assert result is None + # top_domain_under_public_suffix가 비어서 no_domain이거나, encode 실패로 invalid_domain + assert error in {"no_domain", "invalid_domain"} From b186e34ab48a668d6f5711259537d00f540c1617 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:06:14 +0900 Subject: [PATCH 068/173] =?UTF-8?q?[Feat]=20=EB=8F=84=EB=A9=94=EC=9D=B8=20?= =?UTF-8?q?=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20=EC=98=A4=EC=BC=80?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=EB=A0=88=EC=9D=B4=ED=84=B0=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20(=EC=8B=A0=ED=98=B8=20=ED=95=A9=EC=82=B0=C2=B7score?= =?UTF-8?q?=20cap=20=EC=A0=81=EC=9A=A9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/domain_heuristic/check.py | 82 ++++++++++++ tests/services/domain_heuristic/test_check.py | 119 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 app/services/domain_heuristic/check.py create mode 100644 tests/services/domain_heuristic/test_check.py diff --git a/app/services/domain_heuristic/check.py b/app/services/domain_heuristic/check.py new file mode 100644 index 0000000..74a6a52 --- /dev/null +++ b/app/services/domain_heuristic/check.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from urllib.parse import urlparse + +import tldextract + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.services.domain_heuristic.dga import check_dga +from app.services.domain_heuristic.patterns import check_patterns +from app.services.domain_heuristic.rdap import lookup_rdap +from app.services.domain_heuristic.typosquatting import check_typosquatting + +logger = get_logger(__name__) + + +# 모듈 로드 시 1회 구성 — 호출당 dict 재생성 비용 제거. settings hot-reload는 사용하지 않는다. +_SIGNAL_SCORES: dict[DomainHeuristicSignal, int] = { + DomainHeuristicSignal.IP_DIRECT: settings.score_weight_ip_direct, + DomainHeuristicSignal.TYPO_DOMAIN: settings.score_weight_typo_domain, + DomainHeuristicSignal.PUNYCODE_IDN: settings.score_weight_punycode_idn, + DomainHeuristicSignal.NEW_DOMAIN: settings.score_weight_new_domain, + DomainHeuristicSignal.SUBDOMAIN_OVERUSE: settings.score_weight_subdomain_overuse, + DomainHeuristicSignal.NO_HTTPS: settings.score_weight_no_https, + DomainHeuristicSignal.OPEN_REDIRECT_PARAM: settings.score_weight_open_redirect_param, + DomainHeuristicSignal.HYPHEN_OVERUSE: settings.score_weight_hyphen_overuse, + DomainHeuristicSignal.SUSPICIOUS_TLD: settings.score_weight_suspicious_tld, + DomainHeuristicSignal.DGA_LIKE: settings.score_weight_dga_like, + DomainHeuristicSignal.HOSTING_PLATFORM: settings.score_weight_hosting_platform, +} + + +async def check_domain_heuristic(url: str) -> DomainHeuristicResult: + ext = tldextract.extract(url) + domain = ext.top_domain_under_public_suffix or (urlparse(url).hostname or "") + + signals: list[DomainHeuristicSignal] = [] + rdap_info = None + rdap_error = None + + try: + pattern_signals = check_patterns(url) + signals.extend(pattern_signals) + except Exception as exc: + logger.warning("domain_heuristic.patterns_error", error=str(exc)) + + # IP 직접 접근이면 도메인 기반 신호 분석 불필요 + if DomainHeuristicSignal.IP_DIRECT not in signals: + try: + dga_signal = check_dga(url) + if dga_signal: + signals.append(dga_signal) + except Exception as exc: + logger.warning("domain_heuristic.dga_error", error=str(exc)) + + # 호스팅 플랫폼(github.io, netlify.app 등)은 suffix가 브랜드의 suffix와 + # 다른 것이 정상 → 타이포스쿼팅 판정에서 제외 + if DomainHeuristicSignal.HOSTING_PLATFORM not in signals: + try: + typo_signal = check_typosquatting(url) + if typo_signal: + signals.append(typo_signal) + except Exception as exc: + logger.warning("domain_heuristic.typo_error", error=str(exc)) + + # lookup_rdap은 내부에서 모든 예외를 error_code로 변환하므로 재래핑 불필요 + rdap_info, rdap_error = await lookup_rdap(url) + if rdap_info and rdap_info.is_new_domain: + signals.append(DomainHeuristicSignal.NEW_DOMAIN) + + # 휴리스틱은 보조 시그널이라 합산이 GSB/URLhaus를 압도하지 않도록 캡 적용 + raw_score = sum(_SIGNAL_SCORES.get(s, 0) for s in signals) + score = min(raw_score, settings.domain_heuristic_score_cap) + + return DomainHeuristicResult( + domain=domain, + score=score, + signals=signals, + rdap=rdap_info, + rdap_error=rdap_error, + ) diff --git a/tests/services/domain_heuristic/test_check.py b/tests/services/domain_heuristic/test_check.py new file mode 100644 index 0000000..db9b9a6 --- /dev/null +++ b/tests/services/domain_heuristic/test_check.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from datetime import UTC +from unittest.mock import AsyncMock, patch + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo +from app.services.domain_heuristic.check import check_domain_heuristic + +_RDAP_PATH = "app.services.domain_heuristic.check.lookup_rdap" + + +@pytest.mark.asyncio +async def test_ip_direct_url(): + result = await check_domain_heuristic("http://192.168.1.1/login") + assert isinstance(result, DomainHeuristicResult) + assert DomainHeuristicSignal.IP_DIRECT in result.signals + assert result.score == 40 # IP_DIRECT만 발동 + assert result.rdap is None # IP면 RDAP 조회 안 함 + + +@pytest.mark.asyncio +async def test_score_accumulates(): + # http + .xyz TLD → NO_HTTPS(30) + SUSPICIOUS_TLD(20) = 50 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + result = await check_domain_heuristic("http://example.xyz/") + + assert DomainHeuristicSignal.NO_HTTPS in result.signals + assert DomainHeuristicSignal.SUSPICIOUS_TLD in result.signals + assert result.score == 50 + + +@pytest.mark.asyncio +async def test_rdap_failure_does_not_break_pipeline(): + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "timeout") + result = await check_domain_heuristic("https://example.com/") + + assert isinstance(result, DomainHeuristicResult) + assert result.rdap is None + assert result.rdap_error == "timeout" + assert DomainHeuristicSignal.NEW_DOMAIN not in result.signals + + +@pytest.mark.asyncio +async def test_new_domain_signal_from_rdap(): + from datetime import datetime + + fake_rdap = RdapInfo( + domain="example.com", + registrar="TestReg", + created_date=datetime(2026, 4, 15, tzinfo=UTC), + expiry_date=None, + domain_age_days=5, + is_new_domain=True, + ) + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (fake_rdap, None) + result = await check_domain_heuristic("https://example.com/") + + assert DomainHeuristicSignal.NEW_DOMAIN in result.signals + assert result.rdap == fake_rdap + + +@pytest.mark.asyncio +async def test_result_domain_field(): + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "timeout") + result = await check_domain_heuristic("https://login.naver.com/") + + assert result.domain == "naver.com" + + +@pytest.mark.asyncio +async def test_hosting_platform_skips_typosquatting(): + # github.io / netlify.app 같은 호스팅 플랫폼은 타이포스쿼팅 판정에서 제외돼야 함 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + r1 = await check_domain_heuristic("https://user.github.io/repo") + r2 = await check_domain_heuristic("https://myapp.netlify.app/") + + assert DomainHeuristicSignal.HOSTING_PLATFORM in r1.signals + assert DomainHeuristicSignal.TYPO_DOMAIN not in r1.signals + assert DomainHeuristicSignal.HOSTING_PLATFORM in r2.signals + assert DomainHeuristicSignal.TYPO_DOMAIN not in r2.signals + + +@pytest.mark.asyncio +async def test_score_clamped_to_cap(): + # 다중 신호가 캡(80)을 넘어도 응답 score는 캡으로 고정돼야 한다. + # http + .xyz + 4-segment subdomain + open-redirect param + 긴 라벨 + # = NO_HTTPS(30) + SUSPICIOUS_TLD(20) + SUBDOMAIN_OVERUSE(25) + OPEN_REDIRECT_PARAM(20) + # + HYPHEN_OVERUSE(20) = 115 → 80으로 클램프 + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + result = await check_domain_heuristic( + "http://a.b.c.d.averylongsingledomainlabelname.xyz/?redirect=evil" + ) + + assert result.score == 80 + + +@pytest.mark.asyncio +async def test_known_safe_domains_not_malicious(): + # 실제 정상 도메인들이 score_malicious_threshold(50) 미만으로 분류돼야 함 + safe_urls = [ + "https://bing.com/", + "https://cnn.com/", + "https://bbc.com/", + "https://mit.edu/", + "https://netflix.com/", + "https://spotify.com/", + ] + with patch(_RDAP_PATH, new_callable=AsyncMock) as mock_rdap: + mock_rdap.return_value = (None, "not_found") + for url in safe_urls: + result = await check_domain_heuristic(url) + assert result.score < 50, f"{url} score={result.score}, signals={result.signals}" From b6586129e86878e023b820495399d08ae488a29c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:06:35 +0900 Subject: [PATCH 069/173] =?UTF-8?q?[Refactor]=20=ED=8C=8C=EC=9D=B4?= =?UTF-8?q?=ED=94=84=EB=9D=BC=EC=9D=B8/=EB=B6=84=EC=84=9D=20=EC=8A=A4?= =?UTF-8?q?=ED=82=A4=EB=A7=88=EC=97=90=20=EB=8F=84=EB=A9=94=EC=9D=B8=20?= =?UTF-8?q?=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20=EB=8B=A8=EA=B3=84=20?= =?UTF-8?q?=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 4 ++++ app/schemas/pipeline.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 3302fcc..8788a1e 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -1,5 +1,6 @@ # 하위 호환 re-export. 신규 코드는 각 모듈에서 직접 import할 것. from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo from app.schemas.normalize import NormalizeResult from app.schemas.pipeline import ( PipelineFailure, @@ -14,6 +15,8 @@ __all__ = [ "AnalyzeAccepted", "AnalyzeRequest", + "DomainHeuristicResult", + "DomainHeuristicSignal", "GSBMatch", "GSBResult", "HopRecord", @@ -23,6 +26,7 @@ "PipelineStage", "PipelineStages", "PipelineSuccess", + "RdapInfo", "ThreatDbResult", "URLhausResult", "UnchainResult", diff --git a/app/schemas/pipeline.py b/app/schemas/pipeline.py index 9a74ce4..eb652ed 100644 --- a/app/schemas/pipeline.py +++ b/app/schemas/pipeline.py @@ -3,6 +3,7 @@ from pydantic import BaseModel +from app.schemas.domain_heuristic import DomainHeuristicResult from app.schemas.normalize import NormalizeResult from app.schemas.threat_db import ThreatDbResult from app.schemas.unchain import UnchainResult @@ -12,6 +13,7 @@ class PipelineStage(str, Enum): NORMALIZE = "normalize" UNCHAIN = "unchain" THREAT_DB = "threat_db" + DOMAIN_HEURISTIC = "domain_heuristic" class PipelineStages(BaseModel): @@ -20,6 +22,7 @@ class PipelineStages(BaseModel): normalize: NormalizeResult unchain: UnchainResult threat_db: ThreatDbResult + domain_heuristic: DomainHeuristicResult class PipelineSuccess(BaseModel): From 41ade7f0600f6e7d8c4b0177c04c92c791c3c4b0 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:06:57 +0900 Subject: [PATCH 070/173] =?UTF-8?q?[Feat]=20pipeline=EC=97=90=20=EB=8F=84?= =?UTF-8?q?=EB=A9=94=EC=9D=B8=20=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20?= =?UTF-8?q?=EB=8B=A8=EA=B3=84=20=EC=97=B0=EA=B2=B0=20=EB=B0=8F=20=ED=86=B5?= =?UTF-8?q?=ED=95=A9=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/pipeline.py | 39 +++++++++++++-- tests/services/test_pipeline.py | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 tests/services/test_pipeline.py diff --git a/app/services/pipeline.py b/app/services/pipeline.py index 23c8c5d..b1b300c 100644 --- a/app/services/pipeline.py +++ b/app/services/pipeline.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.exceptions import NormalizationError +from app.schemas.domain_heuristic import DomainHeuristicResult from app.schemas.normalize import NormalizeResult from app.schemas.pipeline import ( PipelineFailure, @@ -15,6 +16,7 @@ ) from app.schemas.threat_db import ThreatDbResult from app.schemas.unchain import UnchainResult +from app.services.domain_heuristic import check_domain_heuristic from app.services.normalizer import normalize_url from app.services.threat_db import check_threat_db from app.services.unchainer import unchain_url @@ -30,13 +32,36 @@ def _stage_normalize(log: structlog.stdlib.BoundLogger, original_url: str) -> No async def _stage_unchain(log: structlog.stdlib.BoundLogger, normalized_url: str) -> UnchainResult: result = await unchain_url(normalized_url) - log.info("pipeline.unchain.done", final_url=result.final_url, hops=result.hop_count, signals=result.signals) + log.info( + "pipeline.unchain.done", + final_url=result.final_url, + hops=result.hop_count, + signals=result.signals, + ) return result -async def _stage_threat_db(log: structlog.stdlib.BoundLogger, final_url: str, session: AsyncSession) -> ThreatDbResult: +async def _stage_threat_db( + log: structlog.stdlib.BoundLogger, final_url: str, session: AsyncSession +) -> ThreatDbResult: result = await check_threat_db(session, final_url) - log.info("pipeline.threat_db.done", is_malicious=result.is_malicious, sources_checked=result.sources_checked) + log.info( + "pipeline.threat_db.done", + is_malicious=result.is_malicious, + sources_checked=result.sources_checked, + ) + return result + + +async def _stage_domain_heuristic( + log: structlog.stdlib.BoundLogger, final_url: str +) -> DomainHeuristicResult: + result = await check_domain_heuristic(final_url) + log.info( + "pipeline.domain_heuristic.done", + score=result.score, + signals=[s.value for s in result.signals], + ) return result @@ -61,11 +86,17 @@ async def run_pipeline( unchain = await _stage_unchain(log, norm.normalized_url) threat = await _stage_threat_db(log, unchain.final_url, session) + heuristic = await _stage_domain_heuristic(log, unchain.final_url) log.info("pipeline.done", final_url=unchain.final_url) return PipelineSuccess( analysis_id=analysis_id, original_url=original_url, final_url=unchain.final_url, - stages=PipelineStages(normalize=norm, unchain=unchain, threat_db=threat), + stages=PipelineStages( + normalize=norm, + unchain=unchain, + threat_db=threat, + domain_heuristic=heuristic, + ), ) diff --git a/tests/services/test_pipeline.py b/tests/services/test_pipeline.py new file mode 100644 index 0000000..4f79265 --- /dev/null +++ b/tests/services/test_pipeline.py @@ -0,0 +1,87 @@ +"""run_pipeline 통합 회귀 — stage 추가 시 응답 스키마/순서 검증.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import pytest +from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal +from app.schemas.normalize import NormalizeResult +from app.schemas.pipeline import PipelineFailure, PipelineStage, PipelineSuccess +from app.schemas.threat_db import GSBResult, ThreatDbResult, URLhausResult +from app.schemas.unchain import UnchainResult +from app.services.pipeline import run_pipeline +from sqlalchemy.ext.asyncio import AsyncSession + + +def _make_unchain(final_url: str) -> UnchainResult: + return UnchainResult(input_url=final_url, final_url=final_url, hops=[], hop_count=0, signals=[]) + + +def _make_threat(final_url: str) -> ThreatDbResult: + return ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=2, + gsb=GSBResult(checked=True), + urlhaus=URLhausResult(checked=True), + ) + + +def _make_heuristic(domain: str) -> DomainHeuristicResult: + return DomainHeuristicResult( + domain=domain, + score=15, + signals=[DomainHeuristicSignal.HOSTING_PLATFORM], + rdap=None, + rdap_error="not_found", + ) + + +@pytest.mark.asyncio +async def test_run_pipeline_includes_domain_heuristic_stage(async_session: AsyncSession) -> None: + """domain_heuristic stage 추가가 응답 스키마에 정상 반영되는지 검증.""" + final_url = "https://example.com/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _make_heuristic("example.com") + + result = await run_pipeline("aid-1", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + assert result.analysis_id == "aid-1" + assert result.final_url == final_url + assert result.stages.domain_heuristic.domain == "example.com" + assert result.stages.domain_heuristic.signals == [DomainHeuristicSignal.HOSTING_PLATFORM] + # threat_db → domain_heuristic 순서 — heuristic 인자는 unchain.final_url + mock_heuristic.assert_awaited_once_with(final_url) + + +@pytest.mark.asyncio +async def test_run_pipeline_normalize_failure_skips_heuristic( + async_session: AsyncSession, +) -> None: + """normalize 단계 실패 시 후속 단계가 호출되지 않아야 한다.""" + from app.core.exceptions import NormalizationError + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock) as mock_h, + ): + mock_norm.side_effect = NormalizationError("invalid") + + result = await run_pipeline("aid-2", "not a url", async_session) + + assert isinstance(result, PipelineFailure) + assert result.failed_at_stage == PipelineStage.NORMALIZE + mock_h.assert_not_awaited() From 9b8f53a9bcc82c3b9ef25024b8174f3848295f6c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:07:15 +0900 Subject: [PATCH 071/173] =?UTF-8?q?[Refactor]=20lifespan=EC=97=90=20RDAP?= =?UTF-8?q?=20client=20=EC=A2=85=EB=A3=8C=20=ED=9B=85=20=EC=97=B0=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/main.py b/app/main.py index 8e7a79e..b67c983 100644 --- a/app/main.py +++ b/app/main.py @@ -12,6 +12,7 @@ from app.core.scheduler import shutdown_scheduler, start_scheduler from app.db.session import engine from app.middleware.request_context import RequestContextMiddleware +from app.services.domain_heuristic.rdap import aclose_client as aclose_rdap_client logger = get_logger(__name__) @@ -59,6 +60,7 @@ def _log_startup_sync_result(t: asyncio.Task) -> None: timeout=2.0, ) shutdown_scheduler(wait=False) + await aclose_rdap_client() await engine.dispose() From f0e9aa8210c1a6ba543d1b3c654a0f0e0bcd9f41 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:07:31 +0900 Subject: [PATCH 072/173] =?UTF-8?q?[Docs]=20README=EC=97=90=203=EB=8B=A8?= =?UTF-8?q?=EA=B3=84=20=EB=8F=84=EB=A9=94=EC=9D=B8=20=ED=9C=B4=EB=A6=AC?= =?UTF-8?q?=EC=8A=A4=ED=8B=B1=20=EC=A0=90=EC=88=98=ED=91=9C=20=EB=B0=8F=20?= =?UTF-8?q?HOSTING=5FPLATFORM=20=EC=9D=98=EB=AF=B8=20=EC=84=A4=EB=AA=85=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 06b5f8d..f48557a 100644 --- a/README.md +++ b/README.md @@ -246,18 +246,21 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 | 검사 항목 | 위험 신호 예시 | 점수 | |----------|----------------|------| -| 오타 도메인 (레벤슈타인 거리 1~2) | `naverr.com`, `naaver.com` | +30 | -| 신규 도메인 (RDAP 등록 30일 미만) | `created_date` 기준 | +25 | | IP 직접 접근 | `http://192.168.x.x/login` | +40 | +| 오타 도메인 (레벤슈타인 거리 1~2) | `naverr.com`, `naaver.com` | +40 | +| punycode / IDN 호모글리프 | `xn--naver-xxx.com` | +35 | +| HTTPS 미사용 | `http://` | +30 | +| 신규 도메인 (RDAP 등록 30일 미만) | `created_date` 기준 | +30 | +| 서브도메인 과다 중첩 | `signin.auth.naver.attacker.xyz` | +25 | | 특수문자·하이픈 과다 | `login-secure-naver-auth.com` | +20 | -| 서브도메인 과다 중첩 | `signin.auth.naver.attacker.xyz` | +35 | -| HTTPS 미사용 (로그인 페이지) | `http://` | +15 | -| 합법 호스팅 플랫폼 (가산) | `github.io`, `netlify.app` | +15 | -| 오픈 리다이렉트 파라미터 | `?url=`, `?redirect=` | +35 | - -레벤슈타인 거리 함수는 외부 라이브러리에 의존하지 않고 직접 구현합니다 (DP). -RDAP 응답은 도메인 단위로 인메모리 캐싱(24h) 하여 동일 도메인 재조회 비용을 -줄입니다. +| 의심 TLD | `.zip`, `.mov`, `.xyz`, `.top` 등 | +20 | +| 오픈 리다이렉트 파라미터 | `?url=`, `?redirect=` | +20 | +| DGA 의심 도메인 | Shannon 엔트로피 ≥ 3.5 또는 자음 비율 ≥ 0.7 | +15 | +| 합법 호스팅 플랫폼 (공유 호스팅 주의 가중치) | `user.github.io`, `app.netlify.app` | +15 | + +레벤슈타인 거리 함수는 외부 라이브러리에 의존하지 않고 직접 구현합니다 (DP). 약 500개 브랜드 화이트리스트(`brands.txt`)와 비교합니다. DGA 탐지는 Shannon 엔트로피와 자음 비율 통계만 사용하며 외부 모델이 필요 없습니다. RDAP 응답은 도메인 단위로 인메모리 캐싱(24h) 하여 동일 도메인 재조회 비용을 줄입니다. RDAP 실패 시 신생 도메인 신호를 발동하지 않습니다 ("모름"을 "위험"으로 취급하지 않는 원칙). + +`HOSTING_PLATFORM` 은 "이 도메인이 악성이다" 라는 신호가 아니라 **공유 호스팅 컨텍스트**(GitHub Pages·Netlify·Vercel·Heroku 등 다수 테넌트가 같은 상위 도메인을 공유)를 나타내는 주의 가중치입니다. URLhaus 매칭 키가 `host + path-prefix` 로 확장되는 것과 같은 맥락에서, 계정·리포 단위로 악성 여부가 갈리는 환경이므로 +15 를 가산합니다. 플랫폼 루트 도메인 자체(`netlify.app`, `vercel.app` 등)는 정상 운영 도메인이므로 타이포스쿼팅 검사에서 제외됩니다. ### 4단계 — 페이지 콘텐츠 정적 분석 From 988d3eeee17e86727882be19f31e7d22265a3d16 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:13:20 +0900 Subject: [PATCH 073/173] =?UTF-8?q?[Chore]=20=EB=8F=84=EB=A9=94=EC=9D=B8?= =?UTF-8?q?=20=ED=9C=B4=EB=A6=AC=EC=8A=A4=ED=8B=B1=20=EB=B6=84=EC=84=9D=20?= =?UTF-8?q?DEMO=20=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/demo_domain_heuristic.py | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/demo/demo_domain_heuristic.py diff --git a/tests/demo/demo_domain_heuristic.py b/tests/demo/demo_domain_heuristic.py new file mode 100644 index 0000000..57cd8b0 --- /dev/null +++ b/tests/demo/demo_domain_heuristic.py @@ -0,0 +1,84 @@ +""" +RDAP 조회는 실제 네트워크(`rdap.org` bootstrap)로 나감. +환경에 따라 일부 도메인은 RDAP 서버가 응답하지 않을 수 있으며, +그 경우 `rdap_error` 만 찍히고 파이프라인은 그대로 진행됨. +""" + +from __future__ import annotations + +import asyncio + +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.services.domain_heuristic import check_domain_heuristic +from app.services.domain_heuristic.rdap import aclose_client + +DIVIDER = "-" * 72 + + +CASES: list[tuple[str, str]] = [ + # --- 정상 도메인 (신호 없음 예상) --- + ("정상 도메인 (google)", "https://www.google.com/"), + ("정상 도메인 (naver)", "https://www.naver.com/"), + # --- 패턴 기반 신호 --- + ("IP 직접 접근", "http://192.168.1.1/login"), + ("HTTPS 미사용", "http://example.com/"), + ("의심 TLD (.xyz)", "https://free-money.xyz/"), + ("의심 TLD (.zip)", "https://invoice.zip/"), + ("하이픈 과다 + 긴 레이블", "https://login-secure-naver-auth-verify.com/"), + ("서브도메인 과다 중첩", "https://signin.auth.account.verify.example.com/"), + ("Punycode / IDN", "https://xn--nver-9na.com/"), + ("오픈 리다이렉트 파라미터", "https://example.com/go?url=http://evil.xyz"), + # --- 호스팅 플랫폼 (서브도메인 경유 vs 플랫폼 루트) --- + ("호스팅 플랫폼 경유 (github.io)", "https://someuser.github.io/repo/"), + ("호스팅 플랫폼 경유 (netlify.app)", "https://myapp.netlify.app/"), + ("호스팅 플랫폼 루트 (typo 오탐 방어)", "https://netlify.app/"), + # --- 타이포스쿼팅 --- + ("타이포스쿼팅 (distance 1)", "https://naverr.com/"), + ("타이포스쿼팅 (suffix 상이)", "https://naver.net/"), + # --- DGA 유사 --- + ("DGA 유사 (무작위 문자열)", "https://xjqkzvnpwmrbtld.com/"), + # --- RDAP 신규 도메인 (실제 신규 등록 도메인이 있을 때만 NEW_DOMAIN 발동) --- + ("RDAP 조회 (기성 도메인)", "https://www.cloudflare.com/"), +] + + +def _fmt_rdap(result: DomainHeuristicResult) -> str: + if result.rdap_error: + return f"error={result.rdap_error}" + if result.rdap is None: + return "없음" + rdap = result.rdap + return ( + f"registrar={rdap.registrar!r} " + f"age_days={rdap.domain_age_days} " + f"is_new={rdap.is_new_domain}" + ) + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Domain Heuristic Demo") + print("=" * 72) + + try: + for i, (desc, url) in enumerate(CASES, 1): + print(f"\n [{i:02d}] {desc}") + print(f" input : {url}") + + result = await check_domain_heuristic(url) + + print(f" domain : {result.domain}") + print(f" score : {result.score}") + print(f" signals: {[s.value for s in result.signals]}") + print(f" rdap : {_fmt_rdap(result)}") + print(DIVIDER) + finally: + # 데모 종료 시 RDAP httpx client 정리 (앱 lifespan 외부에서 실행되므로) + await aclose_client() + + print() + + +if __name__ == "__main__": + asyncio.run(main()) From 21798602343320d383be7c2e39ab493437bb294c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 15:54:16 +0900 Subject: [PATCH 074/173] =?UTF-8?q?[Chore]=20unchainer=EC=97=90=EC=84=9C?= =?UTF-8?q?=20=EC=9E=90=EB=AA=85=ED=95=9C=20=EB=84=A4=EB=B9=84=EA=B2=8C?= =?UTF-8?q?=EC=9D=B4=EC=85=98=20=EC=A3=BC=EC=84=9D=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/unchainer/unchain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/services/unchainer/unchain.py b/app/services/unchainer/unchain.py index 467fccb..39a3fb5 100644 --- a/app/services/unchainer/unchain.py +++ b/app/services/unchainer/unchain.py @@ -16,7 +16,6 @@ from app.core.config import settings from app.schemas.analysis import HopRecord, UnchainResult -# 리다이렉트로 간주할 상태 코드 _REDIRECT_CODES: frozenset[int] = frozenset({301, 302, 303, 307, 308}) # 리다이렉트 대상으로 허용하는 스킴 (javascript:, data: 등 차단) @@ -106,7 +105,6 @@ async def _unchain_url_inner(url: str) -> UnchainResult: cookies=None, ) as client: for _ in range(settings.unchain_max_hops): - # 무한 루프 감지 if current_url in visited: signals.append("redirect_loop") break @@ -201,7 +199,6 @@ async def _follow_one_hop( ) return hop, None, f"server_error_{resp.status_code}" - # 리다이렉트 처리 if resp.status_code in _REDIRECT_CODES: raw_location = resp.headers.get("location") if not raw_location: From 06630cbec7bd83cf276287ae9a6e76b60811bb0e Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 21 Apr 2026 16:20:31 +0900 Subject: [PATCH 075/173] =?UTF-8?q?[Docs]=20=EC=BA=90=EC=8B=B1=20=EC=A0=84?= =?UTF-8?q?=EB=9E=B5=20=EB=B3=80=EA=B2=BD=EC=9C=BC=EB=A1=9C=20=EC=9D=B8?= =?UTF-8?q?=ED=95=9C=20README=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f48557a..48f8938 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ | HTTP Client | **httpx** | 외부 API 호출 (GSB / RDAP / Claude / Spring 콜백) | | Crawler | **BeautifulSoup4 + requests** | 페이지 본문 추출, 피싱 신호 탐지 | | Domain Lookup | **RDAP (httpx)** | 도메인 등록일·만료일·레지스트라 조회 | -| Async Queue | **Celery + Redis** | 무거운 분석 작업 큐잉, TTL 캐시 | -| Scheduler | **APScheduler** | URLhaus 주기 동기화, 링크 생존 모니터링 | +| 캐시 | **인메모리 dict + TTL + single-flight** / **SQLite 스냅샷** / **`functools.lru_cache`** | RDAP 24h 캐시·동시 요청 합치기 / URLhaus 로컬 캐시 / Settings 싱글톤 | +| Scheduler | **APScheduler** | URLhaus 주기 동기화 | | Validation | **Pydantic v2 + pydantic-settings** | 요청·응답·환경변수 | | Logging | **structlog** | 구조적 로깅 + request_id 자동 바인딩 | | Lint / Format | **Ruff** | E/F/I/B/SIM/S/UP 룰셋 | @@ -258,7 +258,7 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 | DGA 의심 도메인 | Shannon 엔트로피 ≥ 3.5 또는 자음 비율 ≥ 0.7 | +15 | | 합법 호스팅 플랫폼 (공유 호스팅 주의 가중치) | `user.github.io`, `app.netlify.app` | +15 | -레벤슈타인 거리 함수는 외부 라이브러리에 의존하지 않고 직접 구현합니다 (DP). 약 500개 브랜드 화이트리스트(`brands.txt`)와 비교합니다. DGA 탐지는 Shannon 엔트로피와 자음 비율 통계만 사용하며 외부 모델이 필요 없습니다. RDAP 응답은 도메인 단위로 인메모리 캐싱(24h) 하여 동일 도메인 재조회 비용을 줄입니다. RDAP 실패 시 신생 도메인 신호를 발동하지 않습니다 ("모름"을 "위험"으로 취급하지 않는 원칙). +레벤슈타인 거리 함수는 외부 라이브러리에 의존하지 않고 직접 구현합니다 (DP). 약 500개 브랜드 화이트리스트(`brands.txt`)와 비교합니다. DGA 탐지는 Shannon 엔트로피와 자음 비율 통계만 사용하며 외부 모델이 필요 없습니다. RDAP 응답은 도메인 단위로 인메모리 캐싱(24h, `rdap_cache_ttl_seconds`)하여 동일 도메인 재조회 비용을 줄입니다. 캐시 만료·미스 순간에도 같은 도메인으로 몰리는 요청은 `_inflight` dict + `asyncio.Future` 로 합쳐(**single-flight / request coalescing**) RDAP 서버로 나가는 HTTP 호출을 1건으로 수렴시킵니다. RDAP 실패 시 신생 도메인 신호를 발동하지 않습니다 ("모름"을 "위험"으로 취급하지 않는 원칙). `HOSTING_PLATFORM` 은 "이 도메인이 악성이다" 라는 신호가 아니라 **공유 호스팅 컨텍스트**(GitHub Pages·Netlify·Vercel·Heroku 등 다수 테넌트가 같은 상위 도메인을 공유)를 나타내는 주의 가중치입니다. URLhaus 매칭 키가 `host + path-prefix` 로 확장되는 것과 같은 맥락에서, 계정·리포 단위로 악성 여부가 갈리는 환경이므로 +15 를 가산합니다. 플랫폼 루트 도메인 자체(`netlify.app`, `vercel.app` 등)는 정상 운영 도메인이므로 타이포스쿼팅 검사에서 제외됩니다. From a95e8f417819f7f0f9ba825c4a77a92b7de5e43a Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:35:40 +0900 Subject: [PATCH 076/173] =?UTF-8?q?[Feat]=20openai=20=C2=B7=20beautifulsou?= =?UTF-8?q?p4=20=EC=9D=98=EC=A1=B4=EC=84=B1=20=EC=B6=94=EA=B0=80=20(4?= =?UTF-8?q?=EB=8B=A8=EA=B3=84=20=EC=BD=98=ED=85=90=EC=B8=A0=20=EC=A0=95?= =?UTF-8?q?=EC=A0=81=20=EB=B6=84=EC=84=9D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a8a64fb..9adffad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ dependencies = [ "tenacity>=9.0.0", "apscheduler>=3.10", "tldextract>=5.1.0", + "beautifulsoup4>=4.12.0", + "openai>=1.60.0", ] [project.optional-dependencies] From 5f0446aef9417406f6608256da88673108bd63a3 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:36:13 +0900 Subject: [PATCH 077/173] =?UTF-8?q?[Feat]=204=EB=8B=A8=EA=B3=84=20?= =?UTF-8?q?=EC=BD=98=ED=85=90=EC=B8=A0=20=EB=B6=84=EC=84=9D=20=EC=84=A4?= =?UTF-8?q?=EC=A0=95=EA=B0=92=20=EC=B6=94=EA=B0=80=20(timeout=20=C2=B7=20U?= =?UTF-8?q?A=20pool=20=C2=B7=20=EC=A0=90=EC=88=98=20=EA=B0=80=EC=A4=91?= =?UTF-8?q?=EC=B9=98=20=C2=B7=20score=5F*=5Fthreshold=20=C2=B7=20AI=20prov?= =?UTF-8?q?ider)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 26 ++++++++++++++++++++ app/core/config.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/.env.example b/.env.example index b43a378..4ca3d03 100644 --- a/.env.example +++ b/.env.example @@ -48,6 +48,32 @@ SCORE_WEIGHT_GSB=50 SCORE_WEIGHT_URLHAUS=50 SCORE_WEIGHT_RDAP=20 SCORE_MALICIOUS_THRESHOLD=50 +# 1~3단계 합산이 이 값 이상이면 콘텐츠 정적 분석(네트워크·AI 비용)을 건너뛴다. +SCORE_DANGER_THRESHOLD=61 + +# ---- Content Analysis (4단계) ---- +CONTENT_FETCH_TIMEOUT_SECONDS=8.0 +CONTENT_FETCH_CONNECT_TIMEOUT_SECONDS=3.0 +CONTENT_FETCH_MAX_BYTES=2097152 +CONTENT_EXTERNAL_LINK_RATIO_THRESHOLD=0.8 + +# 페치 실패 가중치 — timeout/connect_error/http_error 같은 도달 실패에만 적용. +# not_html / too_large / unexpected_redirect 는 정상 컨텐츠·파이프라인 이슈로 보고 가산 없이 시그널만. +SCORE_WEIGHT_CONTENT_FETCH_FAILED=10 + +# ---- AI Provider ---- +# auto : OPENAI_API_KEY 있으면 OpenAIProvider, 없으면 NullAIProvider +# openai : 강제 openai (키 없으면 NullAIProvider 로 폴백 + 경고) +# null : 비활성 (규칙 점수만 사용) +AI_PROVIDER=auto + +# ---- OpenAI ---- +# 비워두면 NullAIProvider — 규칙 기반 점수만 사용. +# 모델은 OpenAI 채팅 계열이면 전부 호환 (gpt-4o-mini / gpt-4o / gpt-4.1-mini 등). 교체는 OPENAI_MODEL 한 줄. +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o-mini +OPENAI_TIMEOUT_SECONDS=10.0 +OPENAI_MAX_OUTPUT_TOKENS=300 # ---- Spring 통신 ---- # secrets.token_hex(32) 로 생성한 사전 공유 키. Spring 과 동일한 값 설정 필수. diff --git a/app/core/config.py b/app/core/config.py index a93af51..11efec5 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -134,6 +134,67 @@ def alembic_database_url(self) -> str: hyphen_count_threshold: int = 3 domain_label_length_threshold: int = 20 + # 페이지 콘텐츠 정적 분석 (4단계) + content_fetch_timeout_seconds: float = 8.0 + content_fetch_connect_timeout_seconds: float = 3.0 + content_fetch_max_bytes: int = 2 * 1024 * 1024 # 2MiB 이상이면 끊고 분석 + content_user_agents: list[str] = Field( + default_factory=lambda: [ + # 분석 엔진이 동일 UA를 반복 노출하면 쉽게 블랙리스트되므로 풀에서 라운드로빈 + ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36" + ), + ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) " + "Version/17.0 Safari/605.1.15" + ), + ( + "Mozilla/5.0 (X11; Linux x86_64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/130.0.0.0 Safari/537.36" + ), + ] + ) + content_external_link_ratio_threshold: float = 0.8 + + # 콘텐츠 분석 점수 + score_weight_brand_impersonation: int = 50 + score_weight_logo_alt_impersonation: int = 30 + score_weight_meta_refresh: int = 20 + score_weight_external_link_overuse: int = 15 + # 도달 실패(timeout/connect_error/HTTP 5xx 등)에 대한 보수적 가산. + # not_html(이미지·PDF), too_large(대용량 정상 페이지), unexpected_redirect(unchainer 누락)는 + # 정상 컨텐츠 또는 파이프라인 정합성 문제로 보고 점수 가산 없이 시그널만 남긴다. + score_weight_content_fetch_failed: int = 10 + score_weight_ai_phishing: int = 40 + score_weight_ai_suspicious: int = 20 + # 4단계 단독 캡 — 컨텐츠 분석 단계 안에서만 적용된다. 전 단계 합산은 별도로 score_total_cap 에서 + # 다시 100 으로 클램프되므로, 여기를 낮춰도 합산 상한이 자동으로 같이 낮아지는 게 아니다. + content_analysis_score_cap: int = 100 + + # 1~3단계 합산이 이 임계 이상이면 콘텐츠 정적 분석(네트워크·AI 비용)을 건너뛴다. + # README 의 danger(61+) 구간과 정합. + score_danger_threshold: int = 61 + # caution(31~60) 구간 진입 임계. 이 값 미만이면 safe. + score_caution_threshold: int = 31 + # 단계별 점수 합산이 100 을 넘지 않도록 종합 점수에서 캡 적용. + score_total_cap: int = 100 + + # 4단계 AI 프로바이더 선택. + # auto : OPENAI_API_KEY 있으면 OpenAIProvider, 없으면 NullAIProvider + # openai : 강제 OpenAIProvider (키 없으면 경고 + NullAIProvider 폴백) + # null : 비활성 (규칙 점수만 사용) + ai_provider: Literal["auto", "openai", "null"] = "auto" + + # OpenAI — 모델 교체는 OPENAI_MODEL 한 줄로 끝난다 (gpt-4o-mini / gpt-4o / gpt-4.1-mini). + openai_api_key: str | None = None + openai_model: str = "gpt-4o-mini" + openai_timeout_seconds: float = 10.0 + openai_max_output_tokens: int = 300 + # Spring 통신 internal_api_key: str spring_internal_url: str = "http://localhost:8080" From c0e00fca4270c8805fda0661ae93d291a89ed1a3 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:36:33 +0900 Subject: [PATCH 078/173] =?UTF-8?q?[Feat]=20=EC=BD=98=ED=85=90=EC=B8=A0=20?= =?UTF-8?q?=EB=B6=84=EC=84=9D=20=EC=8A=A4=ED=82=A4=EB=A7=88=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80=20(ContentAnalysisResult=20=C2=B7=20ContentSignal=20?= =?UTF-8?q?=C2=B7=20AIVerdict=20=C2=B7=20TokenUsage=20=C2=B7=20dev=20view)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/content_analysis.py | 106 ++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 app/schemas/content_analysis.py diff --git a/app/schemas/content_analysis.py b/app/schemas/content_analysis.py new file mode 100644 index 0000000..a07d9d5 --- /dev/null +++ b/app/schemas/content_analysis.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class ContentSignal(StrEnum): + BRAND_IMPERSONATION_FORM = "BRAND_IMPERSONATION_FORM" + LOGO_ALT_IMPERSONATION = "LOGO_ALT_IMPERSONATION" + META_REFRESH = "META_REFRESH" + EXTERNAL_LINK_OVERUSE = "EXTERNAL_LINK_OVERUSE" + # 초기 HTML 이 JS 마운트 셸만 담고 있어 정적 추출로 input/form 판정이 불가한 상태. + # 점수 가산 없이 판정 불가 플래그로만 사용 — AI 프롬프트에도 힌트로 전달된다. + SPA_SHELL = "SPA_SHELL" + FETCH_FAILED = "FETCH_FAILED" + SKIPPED_ALREADY_DANGER = "SKIPPED_ALREADY_DANGER" + + +class AIVerdict(StrEnum): + PHISHING = "phishing" + SUSPICIOUS = "suspicious" + BENIGN = "benign" + + +class TokenUsage(BaseModel): + """AI 호출 1건의 토큰 사용량. 모델 비용 · 프롬프트 튜닝 관측에 쓴다.""" + + prompt_tokens: int = Field(ge=0) + completion_tokens: int = Field(ge=0) + total_tokens: int = Field(ge=0) + + +class ContentAnalysisResult(BaseModel): + final_url: str + fetched: bool + score: int = 0 + signals: list[ContentSignal] = Field( + default_factory=list, + description=( + "분석 결과 신호 코드. fetch 실패는 FETCH_FAILED, 선행 단계 danger 로 4단계가 " + "건너뛰어진 경우는 SKIPPED_ALREADY_DANGER 가 들어간다. 분석 결과 차원의 진실원." + ), + ) + + title: str | None = None + has_password_field: bool = False + has_meta_refresh: bool = False + external_link_ratio: float | None = None + brand_impersonation: bool = False + logo_alt_impersonation: bool = False + # JS 마운트 셸로 판단돼 정적 추출이 결정적이지 않은 상태. 여기서 has_password_field=False 는 + # "폼 없음" 이 아니라 "판정 불가" 로 읽어야 한다. + is_spa_shell: bool = False + + ai_verdict: AIVerdict | None = None + ai_reason: str | None = None + ai_error: str | None = Field( + default=None, + description=( + "AI 호출 실패/비활성 사유. " + "ai_unavailable=API 호출 실패, provider_misconfigured=설정 강제 openai 인데 키 누락, " + "None=정상 호출 또는 NullProvider 정상 동작." + ), + ) + # 실제로 응답한 모델 id — 설정 변경/폴백 상황에서도 추적 가능하도록 응답에 싣는다. + ai_model: str | None = None + ai_token_usage: TokenUsage | None = None + + error: str | None = Field( + default=None, + description=( + "페치/스킵 사유 코드 — fetch 레이어의 진실원. " + "timeout/connect_error/http_error_*/not_html/too_large/unexpected_redirect/" + "blocked_host/unexpected/skipped_already_danger 중 하나. " + "signals 와 동시에 채워질 수 있으나 의미는 다르다 (signals=결과 차원, error=원인 코드)." + ), + ) + + +class FetchStatusView(BaseModel): + ok: bool + status_code: int | None = None + html_length: int = 0 + error: str | None = None + + +class ExtractedFeaturesView(BaseModel): + title: str | None = None + has_password_field: bool = False + has_meta_refresh: bool = False + external_link_ratio: float | None = None + image_alts: list[str] = Field(default_factory=list) + is_spa_shell: bool = False + + +class DevFetchExtractResponse(BaseModel): + """디버그 엔드포인트 전용 — fetch 단계 상태 + 파싱 결과 + HTML preview.""" + + url: str + fetch: FetchStatusView + features: ExtractedFeaturesView | None = None + html_preview: str | None = Field( + default=None, + description="HTML 본문 앞부분(최대 2KB) — 파싱 원본 확인용", + ) From b2585ecda2178b84499e75ef3a1fe6008326afaa Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:37:02 +0900 Subject: [PATCH 079/173] =?UTF-8?q?[Refactor]=20=ED=8C=8C=EC=9D=B4?= =?UTF-8?q?=ED=94=84=EB=9D=BC=EC=9D=B8/=EB=B6=84=EC=84=9D=20=EC=8A=A4?= =?UTF-8?q?=ED=82=A4=EB=A7=88=EC=97=90=20content=5Fanalysis=20=EB=8B=A8?= =?UTF-8?q?=EA=B3=84=20=EB=B0=98=EC=98=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/schemas/analysis.py | 14 ++++++++++++++ app/schemas/pipeline.py | 27 ++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/app/schemas/analysis.py b/app/schemas/analysis.py index 8788a1e..fcbc188 100644 --- a/app/schemas/analysis.py +++ b/app/schemas/analysis.py @@ -1,5 +1,13 @@ # 하위 호환 re-export. 신규 코드는 각 모듈에서 직접 import할 것. from app.schemas.analyze import AnalyzeAccepted, AnalyzeRequest +from app.schemas.content_analysis import ( + AIVerdict, + ContentAnalysisResult, + ContentSignal, + DevFetchExtractResponse, + ExtractedFeaturesView, + FetchStatusView, +) from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal, RdapInfo from app.schemas.normalize import NormalizeResult from app.schemas.pipeline import ( @@ -13,10 +21,16 @@ from app.schemas.unchain import HopRecord, UnchainResult __all__ = [ + "AIVerdict", "AnalyzeAccepted", "AnalyzeRequest", + "ContentAnalysisResult", + "ContentSignal", + "DevFetchExtractResponse", "DomainHeuristicResult", "DomainHeuristicSignal", + "ExtractedFeaturesView", + "FetchStatusView", "GSBMatch", "GSBResult", "HopRecord", diff --git a/app/schemas/pipeline.py b/app/schemas/pipeline.py index eb652ed..0c2fc1f 100644 --- a/app/schemas/pipeline.py +++ b/app/schemas/pipeline.py @@ -1,19 +1,34 @@ -from enum import Enum +from enum import StrEnum from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field +from app.schemas.content_analysis import ContentAnalysisResult from app.schemas.domain_heuristic import DomainHeuristicResult from app.schemas.normalize import NormalizeResult from app.schemas.threat_db import ThreatDbResult from app.schemas.unchain import UnchainResult -class PipelineStage(str, Enum): +class PipelineStage(StrEnum): NORMALIZE = "normalize" UNCHAIN = "unchain" THREAT_DB = "threat_db" DOMAIN_HEURISTIC = "domain_heuristic" + CONTENT_ANALYSIS = "content_analysis" + + +class Verdict(StrEnum): + """종합 판정 — UI 의 색상/카피 분기 기준. + + score 만으로 매핑하되, 외부 위협 DB 매치(`threat_db.is_malicious=True`) 면 + 점수와 무관하게 강제로 DANGER 가 된다. blacklist 매치 = 알려진 악성 URL 이라 + 점수 합산이 임계 미달이어도 verdict 자체는 danger 가 맞다. + """ + + SAFE = "safe" + CAUTION = "caution" + DANGER = "danger" class PipelineStages(BaseModel): @@ -23,13 +38,19 @@ class PipelineStages(BaseModel): unchain: UnchainResult threat_db: ThreatDbResult domain_heuristic: DomainHeuristicResult + content_analysis: ContentAnalysisResult class PipelineSuccess(BaseModel): + """파이프라인 성공 응답. verdict/score 는 stages 합산의 결론이라 응답 상단에 노출 — + 클라이언트가 stages 트리를 파싱하지 않고도 곧장 사용자에게 보여줄 수 있게 한다.""" + status: Literal["success"] = "success" analysis_id: str original_url: str final_url: str + verdict: Verdict + score: int = Field(ge=0, le=100) stages: PipelineStages From 7f80c38100a598238badade338506eea021945c8 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:37:38 +0900 Subject: [PATCH 080/173] =?UTF-8?q?[Feat]=20=ED=8E=98=EC=9D=B4=EC=A7=80=20?= =?UTF-8?q?=ED=8E=98=EC=B9=98=20=EB=AA=A8=EB=93=88=20=EC=B6=94=EA=B0=80=20?= =?UTF-8?q?(httpx=20=EC=8B=B1=EA=B8=80=ED=84=B4=20=C2=B7=20SSRF=201=C2=B72?= =?UTF-8?q?=EC=84=A0=20=C2=B7=20UA=20=EB=9D=BC=EC=9A=B4=EB=93=9C=EB=A1=9C?= =?UTF-8?q?=EB=B9=88=20=C2=B7=20=EB=B3=B8=EB=AC=B8=20max=5Fbytes=20?= =?UTF-8?q?=EC=BA=A1):?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/content_analyzer/fetch.py | 295 +++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 app/services/content_analyzer/fetch.py diff --git a/app/services/content_analyzer/fetch.py b/app/services/content_analyzer/fetch.py new file mode 100644 index 0000000..04aaa78 --- /dev/null +++ b/app/services/content_analyzer/fetch.py @@ -0,0 +1,295 @@ +"""최종 URL의 HTML 본문을 취득한다. + +정적 분석 파이프라인 특성상 리다이렉트는 이미 2단계에서 해소됐다고 가정하고 +follow_redirects=False 로 굳힌다. 3xx 를 되받으면 unchainer 가 놓친 케이스이므로 +에러로 처리해 후속 단계에서 파이프라인 정합성을 따질 수 있게 한다. + +httpx.AsyncClient 는 모듈 레벨 싱글턴으로 재사용한다 — 매 요청마다 TCP/TLS 핸드셰이크를 +새로 여는 비용을 피하기 위함. lifespan 에서 aclose_client() 로 명시 종료한다. + +SSRF 방어 — 호스트가 IP 리터럴이면 사설/loopback/link-local 등이 아닌지 검사하고, +잘 알려진 내부 호스트네임(localhost, 클라우드 메타데이터 서비스)은 차단한다. +호스트네임은 추가로 실제 DNS 해석한 결과 IP 들도 동일 룰로 검증한다 — `evil.com → 127.0.0.1` +처럼 lexical 검사를 통과하는 케이스 차단. 단, getaddrinfo 시점과 connect 시점 사이의 +재해석(완전한 DNS rebind)은 본 1선에서 막지 못하므로 운영 환경에서는 egress 방화벽 +또는 분석 전용 프록시로 잔여 위험을 차단해야 한다. +""" + +from __future__ import annotations + +import asyncio +import ipaddress +import random +import socket +from dataclasses import dataclass +from urllib.parse import urlparse + +import httpx + +from app.core.config import settings +from app.core.logging import get_logger + +logger = get_logger(__name__) + +# 분석 대상이 HTML이 아닌 바이너리(PNG, PDF, ZIP 등)면 정적 분석 자체가 의미 없으므로 조기 컷 +_HTML_CT_KEYWORDS: tuple[str, ...] = ("html", "xhtml") + +# SSRF 방어 — 사용자가 정상 도메인을 위장해 보내도 호스트네임 자체가 내부 자원을 가리키면 차단. +# 클라우드 메타데이터 엔드포인트는 IP 가 169.254.169.254 라 IP 검사로도 대부분 잡히지만, +# DNS alias 만 노출된 케이스 대비 호스트네임도 함께 거부한다 (GCE/AWS/Alibaba 모두 커버). +_BLOCKED_HOSTNAMES: frozenset[str] = frozenset( + { + "localhost", + "ip6-localhost", + "ip6-loopback", + "metadata", + "metadata.google.internal", + "instance-data", + "instance-data.ec2.internal", + } +) + + +@dataclass +class FetchResult: + ok: bool + url: str + status_code: int | None = None + html: str = "" + error: str | None = None + + +# 모듈 레벨 싱글턴 — RDAP client 와 동일 패턴. 첫 요청 때 lazy init, lifespan 에서 aclose. +_client: httpx.AsyncClient | None = None + + +def _build_client() -> httpx.AsyncClient: + timeout = httpx.Timeout( + settings.content_fetch_timeout_seconds, + connect=settings.content_fetch_connect_timeout_seconds, + ) + return httpx.AsyncClient( + timeout=timeout, + follow_redirects=False, + verify=True, + cookies=None, + ) + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None: + _client = _build_client() + return _client + + +async def aclose_client() -> None: + """앱 셧다운 훅에서 호출. 테스트에서도 client 상태 초기화에 쓴다.""" + global _client + if _client is not None: + await _client.aclose() + _client = None + + +def _pick_user_agent() -> str: + pool = settings.content_user_agents + # 풀이 비면 misconfiguration 이라 침묵 폴백 대신 raise — 매번 같은 UA 노출되는 + # 봇 탐지 회피 의도를 운영자가 알지 못한 채 흘려보내지 않게 한다. + if not pool: + raise RuntimeError("content_user_agents pool is empty — set CONTENT_USER_AGENTS") + # UA 로테이션은 보안 속성이 아니라 봇 탐지 회피 — random 으로 충분하다. + return random.choice(pool) # noqa: S311 + + +def _is_ip_literal(host: str) -> bool: + try: + ipaddress.ip_address(host) + except ValueError: + return False + return True + + +def _is_blocked_ip_literal(host: str) -> bool: + """호스트가 IP 리터럴이고 사설/loopback/link-local 등이면 True. IP 아니면 False.""" + try: + ip_obj = ipaddress.ip_address(host) + except ValueError: + return False + return ( + ip_obj.is_private + or ip_obj.is_loopback + or ip_obj.is_link_local + or ip_obj.is_reserved + or ip_obj.is_multicast + or ip_obj.is_unspecified + ) + + +def _is_blocked_host(host: str | None) -> bool: + if not host: + return True + # urlparse().hostname 은 이미 IPv6 brackets 를 벗기지만, 다른 경로로 들어온 호스트 + # (테스트·내부 헬퍼)도 동일 룰을 통과시키도록 strip("[]") 를 한 번 더 둔다. + h = host.strip("[]").lower() + if h in _BLOCKED_HOSTNAMES: + return True + return _is_blocked_ip_literal(h) + + +async def _resolved_addrs_blocked(hostname: str) -> bool: + """hostname 의 모든 A/AAAA 응답을 _is_blocked_ip_literal 로 검증. + + 하나라도 사설/loopback 범위면 True. getaddrinfo 는 동기 호출이라 to_thread 로 오프로드. + 완전한 DNS rebind(이 검증 후 connect 시점에 다른 IP 로 재해석) 는 본 1선에서 막을 수 없다 — + 배포 단의 egress 방화벽으로 보강해야 한다. + """ + loop = asyncio.get_running_loop() + infos = await loop.run_in_executor(None, socket.getaddrinfo, hostname, None) + seen: set[str] = set() + for family, _type, _proto, _canon, sockaddr in infos: + if family not in (socket.AF_INET, socket.AF_INET6): + continue + addr = sockaddr[0] + if not isinstance(addr, str): + continue + # %zone 같은 IPv6 scope id 가 붙어 들어오면 제거 — ip_address() 가 거부한다. + # zone 이 다른 동일 IP 도 같은 정책을 적용하면 되므로 dedup 키에서 zone 은 의도적으로 뺀다. + # link-local(fe80::*) 은 zone 유무와 무관하게 _is_link_local 로 잡혀 차단된다. + ip = addr.split("%", 1)[0] + if ip in seen: + continue + seen.add(ip) + if _is_blocked_ip_literal(ip): + return True + return False + + +def _is_html_content_type(headers: httpx.Headers) -> bool: + ct = headers.get("content-type", "").lower() + return any(kw in ct for kw in _HTML_CT_KEYWORDS) + + +def _content_length_over_cap(headers: httpx.Headers, max_bytes: int) -> bool: + raw = headers.get("content-length") + if raw is None: + return False + try: + return int(raw) > max_bytes + except ValueError: + return False + + +async def _read_capped(resp: httpx.Response, max_bytes: int) -> tuple[bytes, bool]: + """응답 본문을 max_bytes 까지만 수집. 초과 시 (부분 본문, True) 반환.""" + buf = bytearray() + async for chunk in resp.aiter_bytes(): + if len(buf) + len(chunk) > max_bytes: + return bytes(buf), True + buf.extend(chunk) + return bytes(buf), False + + +def _decode(body: bytes, resp: httpx.Response) -> str: + encoding = resp.charset_encoding or "utf-8" + try: + return body.decode(encoding, errors="replace") + except LookupError: + # 서버가 이상한 encoding 을 선언했을 때의 안전망 + return body.decode("utf-8", errors="replace") + + +async def fetch_page(url: str) -> FetchResult: + """단일 GET 요청으로 HTML 본문 취득. 실패해도 raise 하지 않고 FetchResult 로 반환.""" + # SSRF 1선 방어 — 내부망 IP/호스트로 떨어지는 요청을 connect 전에 차단. + parsed = urlparse(url) + if _is_blocked_host(parsed.hostname): + logger.info("content_fetch.blocked_host", url=url, host=parsed.hostname) + return FetchResult(ok=False, url=url, error="blocked_host") + + # SSRF 2선 — 호스트네임이 공인 도메인처럼 보여도 실제로 사설 IP 로 풀릴 수 있다 + # (ex. `evil.com → 127.0.0.1`). 사전 DNS 해석 후 모든 응답 IP 를 동일 룰로 검증. + # IP 리터럴은 1선에서 이미 검증돼 통과된 것 — 같은 IP 를 다시 풀 필요 없다. + if parsed.hostname and not _is_ip_literal(parsed.hostname.strip("[]")): + try: + blocked = await _resolved_addrs_blocked(parsed.hostname) + except OSError: + # 해석 실패는 connect 단계 ConnectError 로 떨어지게 그대로 둔다 — + # 여기서 별도 코드 만들지 않는다. + blocked = False + if blocked: + logger.info("content_fetch.blocked_resolved_ip", url=url, host=parsed.hostname) + return FetchResult(ok=False, url=url, error="blocked_host") + + headers = { + "User-Agent": _pick_user_agent(), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7", + } + max_bytes = settings.content_fetch_max_bytes + + try: + client = _get_client() + req = client.build_request("GET", url, headers=headers) + resp = await client.send(req, stream=True) + try: + # 3xx 는 unchainer 가 이미 해소했어야 하는 신호 — 여기서는 예외로 취급 + if 300 <= resp.status_code < 400: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="unexpected_redirect", + ) + + if resp.status_code >= 400: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error=f"http_error_{resp.status_code}", + ) + + if not _is_html_content_type(resp.headers): + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="not_html", + ) + + if _content_length_over_cap(resp.headers, max_bytes): + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="too_large", + ) + + body, over = await _read_capped(resp, max_bytes) + if over: + return FetchResult( + ok=False, + url=url, + status_code=resp.status_code, + error="too_large", + ) + + html = _decode(body, resp) + return FetchResult(ok=True, url=url, status_code=resp.status_code, html=html) + finally: + await resp.aclose() + except httpx.TimeoutException: + return FetchResult(ok=False, url=url, error="timeout") + except httpx.ConnectError: + return FetchResult(ok=False, url=url, error="connect_error") + except httpx.HTTPError as exc: + logger.warning("content_fetch.http_error", url=url, error=str(exc)) + return FetchResult(ok=False, url=url, error="http_error") + except Exception as exc: + logger.warning( + "content_fetch.unexpected", + url=url, + error=str(exc), + error_type=type(exc).__name__, + ) + return FetchResult(ok=False, url=url, error="unexpected") From 0fc8cc7ea2a9af3850714704b076e3d1e11c670b Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:38:04 +0900 Subject: [PATCH 081/173] =?UTF-8?q?[Feat]=20HTML=20=ED=94=BC=EC=B2=98=20?= =?UTF-8?q?=EC=B6=94=EC=B6=9C=20=EC=B6=94=EA=B0=80=20(title=20=C2=B7=20pas?= =?UTF-8?q?sword=20=C2=B7=20meta=20refresh=20=C2=B7=20=EC=99=B8=EB=B6=80?= =?UTF-8?q?=EB=A7=81=ED=81=AC=EB=B9=84=EC=9C=A8=20=C2=B7=20alt=20=C2=B7=20?= =?UTF-8?q?SPA=20shell)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/content_analyzer/extract.py | 151 +++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 app/services/content_analyzer/extract.py diff --git a/app/services/content_analyzer/extract.py b/app/services/content_analyzer/extract.py new file mode 100644 index 0000000..8ed5d9d --- /dev/null +++ b/app/services/content_analyzer/extract.py @@ -0,0 +1,151 @@ +"""HTML 페이지에서 피싱 신호 후보 필드를 추출한다. + +여기서는 "위험하다"는 판단을 하지 않는다. 단순 추출만 담당하고, +점수화·브랜드 비교는 signals.py 가 맡는다. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup + +# href 가 상대 경로(빈 scheme)이면 base_url 과 urljoin 후 http(s) 로 정규화되므로 +# 여기서 검사 시점엔 이 둘만 보면 충분하다. +_NAV_SCHEMES: frozenset[str] = frozenset({"http", "https"}) + +# 주류 프레임워크의 마운트 컨테이너. 번들이 로드돼야 실제 DOM 이 생긴다. +# - root : React (CRA, Vite React) +# - app : Vue 2/3 CLI 기본 +# - __next : Next.js (SSR 없이 빌드된 경우) +# - __nuxt : Nuxt +# - __layout : SvelteKit 구버전 +# - svelte : Svelte/SvelteKit +# - q-app : Quasar/Qwik +_SPA_MOUNT_IDS: frozenset[str] = frozenset( + {"root", "app", "__next", "__nuxt", "__layout", "svelte", "q-app"} +) + +# Angular 는 id 가 아니라 커스텀 요소 태그로 마운트한다. +_SPA_MOUNT_TAGS: frozenset[str] = frozenset({"app-root", "ng-app"}) + +# alt 무제한 수집 시 alt 수만 개를 박은 페이지에서 signals 의 브랜드 매칭 비용이 선형으로 증폭. +# AI 프롬프트도 어차피 [:10] 으로 자르니 추출 단계에서 상한으로 막아둔다. +_MAX_IMAGE_ALTS = 200 + + +@dataclass +class ExtractedFeatures: + title: str | None = None + has_password_field: bool = False + has_meta_refresh: bool = False + external_link_ratio: float | None = None + image_alts: list[str] = field(default_factory=list) + is_spa_shell: bool = False + + +def _normalize_host(host: str | None) -> str: + if not host: + return "" + host = host.lower() + return host.removeprefix("www.") + + +def _compute_external_link_ratio(soup: BeautifulSoup, base_url: str) -> float | None: + base_host = _normalize_host(urlparse(base_url).hostname) + total = 0 + external = 0 + for a in soup.find_all("a"): + href = a.get("href") + if not href or not isinstance(href, str): + continue + joined = urljoin(base_url, href.strip()) + parsed = urlparse(joined) + if parsed.scheme not in _NAV_SCHEMES: + # mailto: / tel: / javascript: 등 네비게이션이 아닌 링크는 비율 분모에서 제외 + continue + total += 1 + if _normalize_host(parsed.hostname) != base_host: + external += 1 + + if total == 0: + return None + return external / total + + +def _extract_title(soup: BeautifulSoup) -> str | None: + if soup.title is None or soup.title.string is None: + return None + text = soup.title.string.strip() + return text or None + + +def _has_password_field(soup: BeautifulSoup) -> bool: + for inp in soup.find_all("input"): + t = inp.get("type") + if isinstance(t, str) and t.lower() == "password": + return True + return False + + +def _has_meta_refresh(soup: BeautifulSoup) -> bool: + # http-equiv=refresh 만 보고 단정하면 헤더만 무의미하게 박힌 페이지가 잘못 매치된다. + # 실제 자동 리다이렉트 의도가 있으려면 content 도 비어있지 않아야 한다. + for meta in soup.find_all("meta"): + http_equiv = meta.get("http-equiv") + if not (isinstance(http_equiv, str) and http_equiv.lower() == "refresh"): + continue + content = meta.get("content") + if isinstance(content, str) and content.strip(): + return True + return False + + +_SPA_MOUNT_ID_SELECTOR: str = ", ".join(f'div[id="{i}"]' for i in _SPA_MOUNT_IDS) + + +def _detect_spa_shell(soup: BeautifulSoup) -> bool: + """초기 HTML 이 JS 마운트 셸만 담고 있는지 여부. + form/input 이 이미 있으면 SSR 됐거나 완전 정적 페이지이므로 판정 불가 대상 아님. + 그 상태에서 주요 프레임워크 마운트 컨테이너(id/태그)가 있으면 SPA 셸로 본다. + + div 마운트 검사는 CSS 셀렉터로 한 번에 — 모든 div 를 순회하지 않는다. + HTML5 의 id 매칭이 case-sensitive 라 셀렉터로는 정확 매치만 잡고, 대문자 ID 도 + 수용하던 종전 동작은 fallback 루프로 유지한다. + """ + if soup.find("form") is not None or soup.find("input") is not None: + return False + if soup.select_one(_SPA_MOUNT_ID_SELECTOR) is not None: + return True + for div in soup.find_all("div"): + div_id = div.get("id") + if isinstance(div_id, str) and div_id.lower() in _SPA_MOUNT_IDS: + return True + return any(soup.find(tag_name) is not None for tag_name in _SPA_MOUNT_TAGS) + + +def _collect_image_alts(soup: BeautifulSoup) -> list[str]: + alts: list[str] = [] + for img in soup.find_all("img"): + if len(alts) >= _MAX_IMAGE_ALTS: + break + alt = img.get("alt") + if not isinstance(alt, str): + continue + alt = alt.strip() + if alt: + alts.append(alt) + return alts + + +def extract_features(html: str, base_url: str) -> ExtractedFeatures: + soup = BeautifulSoup(html or "", "html.parser") + return ExtractedFeatures( + title=_extract_title(soup), + has_password_field=_has_password_field(soup), + has_meta_refresh=_has_meta_refresh(soup), + external_link_ratio=_compute_external_link_ratio(soup, base_url), + image_alts=_collect_image_alts(soup), + is_spa_shell=_detect_spa_shell(soup), + ) From bb2808053ebfd5f3b7028f7ebe3591a4086e677c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:38:26 +0900 Subject: [PATCH 082/173] =?UTF-8?q?[Feat]=20=EC=BD=98=ED=85=90=EC=B8=A0=20?= =?UTF-8?q?=EB=A3=B0=20=EC=A0=90=EC=88=98=20=EC=B6=94=EA=B0=80=20(?= =?UTF-8?q?=EB=B8=8C=EB=9E=9C=EB=93=9C=20=EC=9C=84=EC=9E=A5=C2=B7=EB=A1=9C?= =?UTF-8?q?=EA=B3=A0=20alt=20=EC=9C=84=EC=9E=A5=C2=B7meta=20refresh=C2=B7?= =?UTF-8?q?=EC=99=B8=EB=B6=80=20=EB=A7=81=ED=81=AC=20=EA=B3=BC=EB=8B=A4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/content_analyzer/signals.py | 142 +++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 app/services/content_analyzer/signals.py diff --git a/app/services/content_analyzer/signals.py b/app/services/content_analyzer/signals.py new file mode 100644 index 0000000..7d65c5d --- /dev/null +++ b/app/services/content_analyzer/signals.py @@ -0,0 +1,142 @@ +"""추출된 피처 + 도메인을 바탕으로 규칙 기반 점수와 신호 코드를 산출한다. + +AI 추론 가산은 analyze.py 에서 합산한다 — 여기서는 순수 규칙 레이어만 담당. + +브랜드 매칭 전략 — ASCII 라벨은 단어 경계(\b) 매칭, 비-ASCII(한국어 등)는 substring fallback. +영문은 토큰 경계가 명확해 'pineapple' 이 'apple', 'naverstore' 가 'naver' 로 잡히는 +substring false-positive 를 경계 매칭으로 차단한다. 형태소 경계가 어려운 한국어/한자 +라벨은 substring 으로 받도록 코드 경로가 준비돼 있다. 다만 현재 brands.txt 는 ASCII +라벨만 들어 있어 substring 분기는 dead path 다 — 한국어 라벨 추가 시 자동 활성화된다. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path + +import tldextract + +from app.core.config import settings +from app.schemas.content_analysis import ContentSignal +from app.services.content_analyzer.extract import ExtractedFeatures + +# typosquatting.py 와 동일한 brands.txt 를 재사용. 경로 변경 시 양쪽을 함께 수정해야 한다. +_BRANDS_FILE = Path(__file__).resolve().parents[1] / "domain_heuristic" / "brands.txt" + +# 짧은 브랜드 라벨(kb, sk 등)은 일반 영단어로 오탐되므로 4자 이상만 매칭에 사용. +_MIN_BRAND_LEN = 4 + + +def _read_brand_labels() -> frozenset[str]: + # BRAND_IMPERSONATION 은 4단계 최고 가중치 시그널 — 라벨이 비면 silent 하게 + # 무력화돼 안전 분석이 약해진다. 여기서 강하게 raise 해 부팅 실패로 만든다. + if not _BRANDS_FILE.exists(): + raise RuntimeError( + f"brand labels file missing: {_BRANDS_FILE}. " + "BRAND_IMPERSONATION/LOGO_ALT_IMPERSONATION 시그널이 무력화된다." + ) + labels: set[str] = set() + for line in _BRANDS_FILE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + ext = tldextract.extract(f"https://{line}") + if ext.domain and len(ext.domain) >= _MIN_BRAND_LEN: + labels.add(ext.domain.lower()) + if not labels: + raise RuntimeError( + f"brand labels file produced zero labels: {_BRANDS_FILE}" + ) + return frozenset(labels) + + +@lru_cache(maxsize=1) +def _brand_label_index() -> tuple[frozenset[str], frozenset[str], re.Pattern[str] | None]: + """라벨을 lazy 하게 1회만 로드해 (ASCII set, non-ASCII set, ASCII alternation 패턴) 으로 반환. + + 예전엔 모듈 임포트 시점에 즉시 raise 하는 방식이라 라벨 파일 경로 변경/스왑 시 + 테스트 픽스처 단계에서 import 자체가 죽는 문제가 있었다. lru_cache 로 첫 호출 + 시점까지 미루고, 테스트는 `_brand_label_index.cache_clear()` 로 리셋하면 된다. + """ + labels = _read_brand_labels() + ascii_labels = frozenset(label for label in labels if label.isascii()) + non_ascii_labels = labels - ascii_labels + pattern = ( + re.compile(r"\b(" + "|".join(re.escape(label) for label in ascii_labels) + r")\b") + if ascii_labels + else None + ) + return ascii_labels, non_ascii_labels, pattern + + +@dataclass +class ContentScoring: + score: int = 0 + signals: list[ContentSignal] = field(default_factory=list) + brand_impersonation: bool = False + logo_alt_impersonation: bool = False + + +def _url_brand_label(final_url: str) -> str: + ext = tldextract.extract(final_url) + return (ext.domain or "").lower() + + +def _brands_in_text(text: str | None) -> set[str]: + # 영문은 \b 경계로만 매치 — 'pineapple' → 'apple', 'naverstore' → 'naver' 같은 + # substring false-positive 차단. 한국어 등 비-ASCII 라벨은 본문에 분리 없이 박히므로 substring. + # 4자 미만 라벨은 로드 단계에서 이미 제외됐다. + if not text: + return set() + _ascii_labels, non_ascii_labels, ascii_pattern = _brand_label_index() + lowered = text.lower() + found: set[str] = set() + if ascii_pattern is not None: + found.update(ascii_pattern.findall(lowered)) + for label in non_ascii_labels: + if label in lowered: + found.add(label) + return found + + +def score_content(features: ExtractedFeatures, final_url: str) -> ContentScoring: + result = ContentScoring() + own_label = _url_brand_label(final_url) + + # SPA 셸은 "판정 불가" 표식일 뿐 — 점수는 주지 않는다. + # 정상 SPA 가 대부분이라 가산하면 오탐이 크다. + if features.is_spa_shell: + result.signals.append(ContentSignal.SPA_SHELL) + + if features.has_password_field: + # 자기 도메인 라벨이 매칭된 브랜드에 있으면 정상. + # 그 외 브랜드가 하나라도 섞이면 위장으로 판단한다. + title_brands = _brands_in_text(features.title) + if title_brands and own_label not in title_brands: + result.brand_impersonation = True + result.signals.append(ContentSignal.BRAND_IMPERSONATION_FORM) + result.score += settings.score_weight_brand_impersonation + + alt_brands: set[str] = set() + for alt in features.image_alts: + alt_brands |= _brands_in_text(alt) + if alt_brands and own_label not in alt_brands: + result.logo_alt_impersonation = True + result.signals.append(ContentSignal.LOGO_ALT_IMPERSONATION) + result.score += settings.score_weight_logo_alt_impersonation + + if features.has_meta_refresh: + result.signals.append(ContentSignal.META_REFRESH) + result.score += settings.score_weight_meta_refresh + + if ( + features.external_link_ratio is not None + and features.external_link_ratio >= settings.content_external_link_ratio_threshold + ): + result.signals.append(ContentSignal.EXTERNAL_LINK_OVERUSE) + result.score += settings.score_weight_external_link_overuse + + result.score = min(result.score, settings.content_analysis_score_cap) + return result From dc89339839723c38d4d18ef2cab884bcebb45617 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:39:12 +0900 Subject: [PATCH 083/173] =?UTF-8?q?[Feat]=20AI=20Provider=20Protocol=20+?= =?UTF-8?q?=20OpenAI=20=EA=B5=AC=ED=98=84=20=EC=B6=94=EA=B0=80=20(Structur?= =?UTF-8?q?ed=20Outputs=20=C2=B7=20Null=20=ED=8F=B4=EB=B0=B1=20=C2=B7=20Ca?= =?UTF-8?q?ncelledError=20re-raise)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/content_analyzer/ai.py | 73 +++++++ app/services/content_analyzer/ai_openai.py | 236 +++++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 app/services/content_analyzer/ai.py create mode 100644 app/services/content_analyzer/ai_openai.py diff --git a/app/services/content_analyzer/ai.py b/app/services/content_analyzer/ai.py new file mode 100644 index 0000000..b41e516 --- /dev/null +++ b/app/services/content_analyzer/ai.py @@ -0,0 +1,73 @@ +"""AI 피싱 추론 어댑터 — 모델을 나중에 갈아끼울 수 있게 Protocol 로 분리. + +모델 미정 상태에서는 NullAIProvider 로 동작. 실제 모델 연동은 +set_ai_provider() 로 부팅 시 런타임에 교체한다 (Spring 측과의 호환성 유지용). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + +from app.schemas.content_analysis import AIVerdict, TokenUsage + + +@dataclass(frozen=True) +class AIPromptContext: + final_url: str + title: str | None + has_password_field: bool + has_meta_refresh: bool + # frozen 데이터클래스는 deep-freeze 가 아니므로 mutable list 를 두면 type 계약과 어긋난다. + # tuple 로 받아 호출 측이 의도치 않게 mutate 하는 경로를 차단. + image_alts: tuple[str, ...] + external_link_ratio: float | None + # True 면 정적 추출이 결정적이지 않음 — has_password_field=False 를 + # "폼 없음" 으로 단정하면 안 된다. + is_spa_shell: bool = False + # 상위 단계(도메인 휴리스틱·threat_db)에서 이미 잡힌 시그널 코드 목록. + # AI 가 페이지 피처 단독으로는 결론 내기 약한 케이스에 사전 정보로 활용한다. + # 예: TYPO_DOMAIN + 브랜드 title + 비밀번호 폼이면 단독 페이지 분석보다 강한 phishing 신호. + upstream_signals: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AIInference: + verdict: AIVerdict + reason: str + # 어떤 모델이 응답했는지 + 토큰 사용량. 응답에 그대로 실어 비용/성능 관측에 쓴다. + # 과거 호환 & 테스트 편의를 위해 옵셔널로 둔다 — 실제 프로바이더는 채워서 돌려준다. + model: str | None = None + token_usage: TokenUsage | None = None + + +class AIProvider(Protocol): + async def infer(self, ctx: AIPromptContext) -> AIInference | None: ... + + +class NullAIProvider: + """모델이 설정되지 않았을 때의 기본 구현. 추론을 시도하지 않고 None 반환. + + `fallback_reason` 이 None 이 아니면 "정상 NullProvider" 가 아니라 다른 프로바이더가 + 부팅 시 폴백된 상태라는 뜻이다. analyze.py 가 이 값을 읽어 응답의 `ai_error` 로 + 노출해 운영자가 NullProvider 정상 동작과 misconfigured 폴백을 구분할 수 있게 한다. + """ + + def __init__(self, *, fallback_reason: str | None = None) -> None: + self.fallback_reason = fallback_reason + + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + return None + + +_provider: AIProvider = NullAIProvider() + + +def get_ai_provider() -> AIProvider: + return _provider + + +def set_ai_provider(provider: AIProvider) -> None: + """앱 부팅 시점에만 호출 — 파이프라인 실행 중 교체하면 race 위험.""" + global _provider + _provider = provider diff --git a/app/services/content_analyzer/ai_openai.py b/app/services/content_analyzer/ai_openai.py new file mode 100644 index 0000000..2e17d9c --- /dev/null +++ b/app/services/content_analyzer/ai_openai.py @@ -0,0 +1,236 @@ +"""OpenAI Chat Completions 기반 피싱 추론 어댑터. + +Structured Outputs (`response_format=json_schema`, `strict=True`) 로 JSON 포맷을 +모델 측에서 강제한다. 클라이언트 측 파싱은 최소한으로 유지하고, 스키마 위반· +API 오류는 전부 None 으로 떨어뜨려 상위 analyze 에서 `ai_error` 로 기록한다. +asyncio.CancelledError 만 예외적으로 re-raise — 취소 신호를 삼키지 않는다. + +모델 교체는 생성자 파라미터 또는 `settings.openai_model` 환경변수로 한다. +OpenAI 채팅 모델(gpt-4o-mini / gpt-4o / gpt-4.1-mini 등)은 전부 같은 코드로 돌아간다. +""" + +from __future__ import annotations + +import asyncio +import json + +from openai import AsyncOpenAI +from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionUserMessageParam, +) +from openai.types.shared_params import ResponseFormatJSONSchema +from openai.types.shared_params.response_format_json_schema import JSONSchema + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.content_analysis import AIVerdict, TokenUsage +from app.services.content_analyzer.ai import AIInference, AIPromptContext + +logger = get_logger(__name__) + +_VERDICT_SCHEMA: JSONSchema = { + "name": "phishing_verdict", + "strict": True, + "schema": { + "type": "object", + "properties": { + "verdict": { + "type": "string", + "enum": [v.value for v in AIVerdict], + }, + "reason": {"type": "string"}, + }, + "required": ["verdict", "reason"], + "additionalProperties": False, + }, +} + +_RESPONSE_FORMAT: ResponseFormatJSONSchema = { + "type": "json_schema", + "json_schema": _VERDICT_SCHEMA, +} + +_SYSTEM_PROMPT = ( + "당신은 웹 페이지 피싱 여부를 판정하는 보안 엔진이다. " + "입력은 이미 크롤러가 뽑아놓은 구조화된 피처(title, 비밀번호 필드 유무, " + "meta refresh 유무, 외부 링크 비율, 이미지 alt 텍스트, 최종 URL, spa_shell 플래그)와 " + "선행 단계(도메인 휴리스틱·위협 DB)에서 이미 잡힌 시그널 코드 배열 upstream_signals 다. " + "규칙 기반 점수는 이미 상위 레이어에서 계산되고 있으므로, 당신은 " + "'이 페이지가 특정 브랜드를 사칭하거나 자격증명을 탈취하려 하는지' 를 " + "뉘앙스 기반으로만 판정한다. " + "upstream_signals 는 도메인/네트워크 레이어의 사전 정보다 — 단독 시그널이 아니라 " + "페이지 피처와 결합해 판정을 강화하는 데 쓴다. 예: TYPO_DOMAIN + 브랜드 title + " + "비밀번호 폼이면 단독 페이지 분석보다 강한 phishing 신호다. NEW_DOMAIN/IP_DIRECT 도 " + "동일한 맥락에서 가중하되, 페이지 측 증거 없이 upstream_signals 만으로 phishing 을 " + "확정하지는 않는다. " + "spa_shell=true 이면 초기 HTML 이 JS 마운트 셸뿐이라 has_password_field=false 는 " + "'폼이 없다' 가 아니라 '정적 추출로는 판정 불가' 를 의미한다. 이 경우 남은 단서" + "(title/URL/이미지 alt/upstream_signals 등)만으로 단정하지 말고 suspicious 또는 benign 을 " + "보수적으로 선택한다. " + "verdict 는 phishing / suspicious / benign 중 하나. reason 은 한국어 1~2문장. " + "확증이 없으면 benign 또는 suspicious 를 쓰고 phishing 은 보수적으로만 사용한다." +) + + +def _build_user_prompt(ctx: AIPromptContext) -> str: + # 토큰 절약 차원에서 짧은 JSON 으로 직렬화. 중요 필드만 추려서 모델이 노이즈에 끌리지 않게 한다. + payload = { + "final_url": ctx.final_url, + "title": ctx.title, + "has_password_field": ctx.has_password_field, + "has_meta_refresh": ctx.has_meta_refresh, + "external_link_ratio": ctx.external_link_ratio, + "spa_shell": ctx.is_spa_shell, + # 과도하게 긴 alt 리스트는 비용만 늘리고 판정에 도움 안 됨 + "image_alts": list(ctx.image_alts[:10]), + # 선행 단계 시그널 — 빈 배열이면 단독 페이지 분석과 동일. + "upstream_signals": list(ctx.upstream_signals), + } + return json.dumps(payload, ensure_ascii=False) + + +def _extract_token_usage(completion: object) -> TokenUsage | None: + # completion.usage 는 OpenAI SDK 가 채워주지만 스트리밍/에러 응답 등에선 비어있을 수 있다. + usage = getattr(completion, "usage", None) + if usage is None: + return None + prompt = getattr(usage, "prompt_tokens", None) + completion_tokens = getattr(usage, "completion_tokens", None) + total = getattr(usage, "total_tokens", None) + if prompt is None or completion_tokens is None or total is None: + return None + return TokenUsage( + prompt_tokens=int(prompt), + completion_tokens=int(completion_tokens), + total_tokens=int(total), + ) + + +def _parse_verdict( + raw: str | None, + *, + model: str, + token_usage: TokenUsage | None, +) -> AIInference | None: + if not raw: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError: + logger.warning("openai_ai.parse_error", raw=raw[:200]) + return None + + verdict_str = data.get("verdict") + reason = data.get("reason") + if not isinstance(verdict_str, str) or not isinstance(reason, str): + return None + + try: + verdict = AIVerdict(verdict_str) + except ValueError: + logger.warning("openai_ai.unknown_verdict", verdict=verdict_str) + return None + + return AIInference( + verdict=verdict, + reason=reason, + model=model, + token_usage=token_usage, + ) + + +class OpenAIProvider: + """OpenAI Chat Completions 기반 AIProvider 구현. + + - 모델/타임아웃/최대 출력 토큰은 생성자에서 오버라이드 가능. 미지정 시 settings 값을 사용. + 같은 클래스로 gpt-4o-mini, gpt-4o, gpt-4.1 등 OpenAI 계열 모델을 바로 스왑할 수 있다. + - API 키가 비어있으면 infer() 는 즉시 None (NullAIProvider 와 동일 동작). + - 요청 실패·파싱 실패는 전부 None 으로 흡수 → 파이프라인은 계속 돈다. + - `asyncio.CancelledError` 는 re-raise. + """ + + def __init__( + self, + *, + model: str | None = None, + timeout_seconds: float | None = None, + max_output_tokens: int | None = None, + ) -> None: + # 생성자 인자가 우선, 없으면 settings 기본값. 인스턴스 생성 후에는 불변으로 본다. + self._model = model if model is not None else settings.openai_model + self._timeout = ( + timeout_seconds if timeout_seconds is not None else settings.openai_timeout_seconds + ) + self._max_output_tokens = ( + max_output_tokens + if max_output_tokens is not None + else settings.openai_max_output_tokens + ) + # 클라이언트는 호출 시점에 지연 생성 — 키가 없으면 아예 만들지 않는다. + self._client: AsyncOpenAI | None = None + # aclose() 후 재사용 방지. 더블 콜은 no-op, 이후 infer() 는 None 반환. + self._closed: bool = False + + @property + def model(self) -> str: + return self._model + + def _get_client(self) -> AsyncOpenAI | None: + if self._closed: + # 정상 시나리오에선 lifespan 종료 후 호출이 없어야 한다. 호출이 들리면 + # provider 재바인딩이 누락된 신호이므로 silent 폴백 대신 한 번 경고로 남긴다. + logger.warning("openai_ai.called_after_close", model=self._model) + return None + if not settings.openai_api_key: + return None + if self._client is None: + self._client = AsyncOpenAI(api_key=settings.openai_api_key) + return self._client + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + if self._client is not None: + await self._client.close() + self._client = None + + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + client = self._get_client() + if client is None: + return None + + messages: list[ChatCompletionMessageParam] = [ + ChatCompletionSystemMessageParam(role="system", content=_SYSTEM_PROMPT), + ChatCompletionUserMessageParam(role="user", content=_build_user_prompt(ctx)), + ] + try: + completion = await client.chat.completions.create( + model=self._model, + messages=messages, + response_format=_RESPONSE_FORMAT, + max_tokens=self._max_output_tokens, + temperature=0, + timeout=self._timeout, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning( + "openai_ai.request_failed", + error=str(exc), + error_type=type(exc).__name__, + model=self._model, + ) + return None + + try: + raw = completion.choices[0].message.content + except (AttributeError, IndexError) as exc: + logger.warning("openai_ai.unexpected_shape", error=str(exc)) + return None + + token_usage = _extract_token_usage(completion) + return _parse_verdict(raw, model=self._model, token_usage=token_usage) From b332e28a0202e9cabfab45bf8dce3a4cedd289c1 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:39:36 +0900 Subject: [PATCH 084/173] =?UTF-8?q?[Feat]=204=EB=8B=A8=EA=B3=84=20?= =?UTF-8?q?=EC=98=A4=EC=BC=80=EC=8A=A4=ED=8A=B8=EB=A0=88=EC=9D=B4=ED=84=B0?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80=20(fetch=C2=B7extract=C2=B7signals=C2=B7A?= =?UTF-8?q?I=20=ED=95=A9=EC=84=B1,=20fetch=20=EC=8B=A4=ED=8C=A8=20?= =?UTF-8?q?=EC=82=AC=EC=9C=A0=EB=B3=84=20=EC=A0=90=EC=88=98=20=EC=A0=95?= =?UTF-8?q?=EC=B1=85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/content_analyzer/__init__.py | 3 + app/services/content_analyzer/analyze.py | 177 ++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 app/services/content_analyzer/__init__.py create mode 100644 app/services/content_analyzer/analyze.py diff --git a/app/services/content_analyzer/__init__.py b/app/services/content_analyzer/__init__.py new file mode 100644 index 0000000..51ad6d9 --- /dev/null +++ b/app/services/content_analyzer/__init__.py @@ -0,0 +1,3 @@ +from app.services.content_analyzer.analyze import analyze_content, skipped_already_danger + +__all__ = ["analyze_content", "skipped_already_danger"] diff --git a/app/services/content_analyzer/analyze.py b/app/services/content_analyzer/analyze.py new file mode 100644 index 0000000..c85813f --- /dev/null +++ b/app/services/content_analyzer/analyze.py @@ -0,0 +1,177 @@ +"""콘텐츠 정적 분석 오케스트레이터 — 4단계 진입점. + +fetch · extract · signals · AI 를 순차 호출하고 ContentAnalysisResult 로 병합한다. +외부 의존성(네트워크·AI) 실패는 degraded 결과로 흡수하되, +asyncio.CancelledError 는 반드시 re-raise 해서 상위 shutdown/timeout 신호가 살아있게 한다. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Iterable + +from app.core.config import settings +from app.core.logging import get_logger +from app.schemas.content_analysis import ( + AIVerdict, + ContentAnalysisResult, + ContentSignal, + TokenUsage, +) +from app.services.content_analyzer.ai import AIPromptContext, AIProvider, get_ai_provider +from app.services.content_analyzer.extract import ExtractedFeatures, extract_features +from app.services.content_analyzer.fetch import fetch_page +from app.services.content_analyzer.signals import ContentScoring, score_content + +logger = get_logger(__name__) + + +def _build_prompt_context( + final_url: str, + features: ExtractedFeatures, + upstream_signals: tuple[str, ...], +) -> AIPromptContext: + return AIPromptContext( + final_url=final_url, + title=features.title, + has_password_field=features.has_password_field, + has_meta_refresh=features.has_meta_refresh, + image_alts=tuple(features.image_alts), + external_link_ratio=features.external_link_ratio, + is_spa_shell=features.is_spa_shell, + upstream_signals=upstream_signals, + ) + + +def _ai_score_weight(verdict: AIVerdict) -> int: + # benign 은 규칙 점수를 감점하지 않는다 — AI 오탐이 정상 점수까지 깎을 수 있기 때문 + if verdict == AIVerdict.PHISHING: + return settings.score_weight_ai_phishing + if verdict == AIVerdict.SUSPICIOUS: + return settings.score_weight_ai_suspicious + return 0 + + +# 정상 컨텐츠(이미지·PDF·대용량 정적 페이지) 또는 파이프라인 정합성 이슈(unchainer 가 놓친 3xx)는 +# 악성 신호로 보지 않는다 — FETCH_FAILED 시그널만 남기고 점수는 가산하지 않는다. +# 그 외(timeout/connect_error/http_error 등)는 보수적으로 settings 의 가중치를 가산. +_FETCH_ERROR_NO_SCORE: frozenset[str] = frozenset( + {"not_html", "too_large", "unexpected_redirect"} +) + + +def _fetch_failed_score(error: str | None) -> int: + if error in _FETCH_ERROR_NO_SCORE: + return 0 + return settings.score_weight_content_fetch_failed + + +def _fetch_failed_result(final_url: str, error: str | None) -> ContentAnalysisResult: + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + score=_fetch_failed_score(error), + signals=[ContentSignal.FETCH_FAILED], + error=error, + ) + + +def skipped_already_danger(final_url: str) -> ContentAnalysisResult: + """선행 단계에서 이미 danger 판정 — 네트워크·AI 비용을 아끼기 위해 분석을 건너뛴 상태. + + 점수는 0으로 두어 상위 합산에 왜곡을 주지 않는다. 이미 다른 신호로 danger 가 + 확정돼 있으므로 여기서 추가 가산할 필요가 없다. + """ + return ContentAnalysisResult( + final_url=final_url, + fetched=False, + score=0, + signals=[ContentSignal.SKIPPED_ALREADY_DANGER], + error="skipped_already_danger", + ) + + +async def analyze_content( + final_url: str, + *, + provider: AIProvider | None = None, + upstream_signals: Iterable[str] = (), +) -> ContentAnalysisResult: + """콘텐츠 정적 분석 진입점. + + provider 를 명시하면 전역 프로바이더 대신 그것을 사용한다 — 모델 비교/디버그용. + None(기본)이면 set_ai_provider() 로 설정된 전역 값을 쓴다. + upstream_signals 는 도메인 휴리스틱·threat_db 등 선행 단계에서 잡힌 시그널 코드. + AI 프롬프트에 사전 정보로 실어 보내 단독 페이지 피처보다 강한 판정이 가능하게 한다. + """ + fetch_result = await fetch_page(final_url) + if not fetch_result.ok: + logger.info( + "content_analysis.fetch_failed", + url=final_url, + error=fetch_result.error, + status=fetch_result.status_code, + ) + return _fetch_failed_result(final_url, fetch_result.error) + + features = extract_features(fetch_result.html, base_url=final_url) + scoring: ContentScoring = score_content(features, final_url) + + ai_verdict: AIVerdict | None = None + ai_reason: str | None = None + ai_error: str | None = None + ai_model: str | None = None + ai_token_usage: TokenUsage | None = None + + active_provider = provider if provider is not None else get_ai_provider() + upstream_tuple = tuple(upstream_signals) + try: + inference = await active_provider.infer( + _build_prompt_context(final_url, features, upstream_tuple) + ) + except asyncio.CancelledError: + # shutdown / 요청 타임아웃 신호는 degraded 결과로 흡수하지 않는다 + raise + except Exception as exc: + logger.warning( + "content_analysis.ai_error", + url=final_url, + error=str(exc), + error_type=type(exc).__name__, + ) + ai_error = "ai_unavailable" + inference = None + + score = scoring.score + if inference is not None: + ai_verdict = inference.verdict + ai_reason = inference.reason + ai_model = inference.model + ai_token_usage = inference.token_usage + score += _ai_score_weight(inference.verdict) + elif ai_error is None: + # 추론이 None 인데 호출 단계 예외도 없었다면 NullAIProvider 동작. + # 부팅 시 misconfiguration 으로 폴백된 NullProvider 면 fallback_reason 을 응답에 노출. + # 정상 NullProvider 는 fallback_reason=None 이라 ai_error 가 None 으로 유지된다. + ai_error = getattr(active_provider, "fallback_reason", None) + + score = min(score, settings.content_analysis_score_cap) + + return ContentAnalysisResult( + final_url=final_url, + fetched=True, + score=score, + signals=list(scoring.signals), + title=features.title, + has_password_field=features.has_password_field, + has_meta_refresh=features.has_meta_refresh, + external_link_ratio=features.external_link_ratio, + brand_impersonation=scoring.brand_impersonation, + logo_alt_impersonation=scoring.logo_alt_impersonation, + is_spa_shell=features.is_spa_shell, + ai_verdict=ai_verdict, + ai_reason=ai_reason, + ai_error=ai_error, + ai_model=ai_model, + ai_token_usage=ai_token_usage, + ) From 6e8870a34ed3741e2812075c95df79341a2cc66c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:39:52 +0900 Subject: [PATCH 085/173] =?UTF-8?q?[Refactor]=20Stage=202=C2=B73=20?= =?UTF-8?q?=EB=B3=91=EB=A0=AC=ED=99=94=20+=20threat=20=EB=A7=A4=EC=B9=98?= =?UTF-8?q?=20=EC=8B=9C=20short-circuit=20=C2=B7=20Stage=204=20=EC=97=B0?= =?UTF-8?q?=EA=B2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/pipeline.py | 187 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 3 deletions(-) diff --git a/app/services/pipeline.py b/app/services/pipeline.py index b1b300c..a5b5ee0 100644 --- a/app/services/pipeline.py +++ b/app/services/pipeline.py @@ -2,10 +2,17 @@ from __future__ import annotations +import asyncio +from contextlib import suppress +from urllib.parse import urlparse + import structlog +import tldextract from sqlalchemy.ext.asyncio import AsyncSession +from app.core.config import settings from app.core.exceptions import NormalizationError +from app.schemas.content_analysis import ContentAnalysisResult from app.schemas.domain_heuristic import DomainHeuristicResult from app.schemas.normalize import NormalizeResult from app.schemas.pipeline import ( @@ -13,9 +20,11 @@ PipelineStage, PipelineStages, PipelineSuccess, + Verdict, ) from app.schemas.threat_db import ThreatDbResult from app.schemas.unchain import UnchainResult +from app.services.content_analyzer import analyze_content, skipped_already_danger from app.services.domain_heuristic import check_domain_heuristic from app.services.normalizer import normalize_url from app.services.threat_db import check_threat_db @@ -65,6 +74,141 @@ async def _stage_domain_heuristic( return result +def _collect_upstream_signals( + threat: ThreatDbResult, heuristic: DomainHeuristicResult +) -> tuple[str, ...]: + """선행 단계 시그널을 한 줄 코드 배열로 평탄화. AI 프롬프트의 사전 정보로 쓰인다. + + threat_db 측은 enum 시그널이 없어 매치 여부를 명시 코드(GSB_THREAT/URLHAUS_THREAT)로 + 합성해 넣는다. 이 단계에 도달했다는 건 short-circuit 미발동 = is_malicious=False 라는 + 뜻이라 두 코드 모두 평소엔 비어있지만, 미래에 약한 매치가 합산만 막혔을 케이스도 + 누락 없이 전달하기 위해 명시적으로 합산한다. + """ + codes: list[str] = [s.value for s in heuristic.signals] + if threat.gsb.is_threat: + codes.append("GSB_THREAT") + if threat.urlhaus.is_threat: + codes.append("URLHAUS_THREAT") + return tuple(codes) + + +async def _stage_content_analysis( + log: structlog.stdlib.BoundLogger, + final_url: str, + upstream_signals: tuple[str, ...], +) -> ContentAnalysisResult: + result = await analyze_content(final_url, upstream_signals=upstream_signals) + log.info( + "pipeline.content_analysis.done", + fetched=result.fetched, + score=result.score, + signals=[s.value for s in result.signals], + ai_verdict=result.ai_verdict.value if result.ai_verdict else None, + upstream_signals=list(upstream_signals), + ) + return result + + +def _preceding_score(threat: ThreatDbResult, heuristic: DomainHeuristicResult) -> int: + """2~3단계 누적 점수. 4단계 건너뛸지 판단하는 기준.""" + score = heuristic.score + if threat.gsb.is_threat: + score += settings.score_weight_gsb + if threat.urlhaus.is_threat: + score += settings.score_weight_urlhaus + return score + + +def _total_score( + threat: ThreatDbResult, + heuristic: DomainHeuristicResult, + content: ContentAnalysisResult, +) -> int: + """전 단계 합산 후 100 으로 캡. content.score 는 4단계가 실제로 돌았을 때만 비-0.""" + return min( + _preceding_score(threat, heuristic) + content.score, + settings.score_total_cap, + ) + + +def _decide_verdict(score: int, threat: ThreatDbResult) -> Verdict: + """blacklist 매치는 점수와 무관하게 danger. 나머지는 점수 구간으로 매핑.""" + if threat.is_malicious: + return Verdict.DANGER + if score >= settings.score_danger_threshold: + return Verdict.DANGER + if score >= settings.score_caution_threshold: + return Verdict.CAUTION + return Verdict.SAFE + + +def _skipped_heuristic(final_url: str) -> DomainHeuristicResult: + """threat match 로 조기 종료된 경우 heuristic 자리에 들어갈 placeholder. + + score=0 이라 합산에 영향 없고, rdap_error 코드로 "우리가 건너뛴 것" 을 식별 가능하게 남긴다. + 응답 스키마의 heuristic 필드가 필수라서 빈 객체라도 채워야 하는 제약 때문. + + domain 값은 정상 경로(check_domain_heuristic)와 동일하게 등록 가능 도메인을 쓴다. + full host 를 쓰면 같은 필드에 의미가 두 가지로 갈리고, 다운스트림에서 + `signin.evil.example.com` 와 `example.com` 이 다른 버킷으로 잡힌다. + """ + ext = tldextract.extract(final_url) + domain = ext.top_domain_under_public_suffix or (urlparse(final_url).hostname or "") + return DomainHeuristicResult( + domain=domain, + score=0, + signals=[], + rdap=None, + rdap_error="skipped_threat_matched", + ) + + +async def _run_stage_2_and_3( + log: structlog.stdlib.BoundLogger, + final_url: str, + session: AsyncSession, +) -> tuple[ThreatDbResult, DomainHeuristicResult, bool]: + """2·3단계를 병렬 실행한다. threat_db 가 먼저 malicious 로 끝나면 heuristic 을 + 즉시 cancel 하고 조기 종료 플래그를 세워 반환한다. + + 반환: (threat, heuristic, short_circuited). + short_circuited=True 면 heuristic 은 placeholder 이고 4단계는 skip 대상. + `CancelledError` 와 stage 내부 예외는 남은 task 를 정리한 뒤 그대로 re-raise. + """ + threat_task = asyncio.create_task(_stage_threat_db(log, final_url, session)) + heur_task = asyncio.create_task(_stage_domain_heuristic(log, final_url)) + + try: + done, _ = await asyncio.wait( + {threat_task, heur_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + + if threat_task in done: + threat = threat_task.result() + if threat.is_malicious: + # RDAP 대기로 5s 가까이 떠 있을 수 있는 heuristic 을 cancel 로 회수. + # danger 확정이니 heuristic 점수는 verdict 에 영향을 주지 못한다. + heur_task.cancel() + with suppress(asyncio.CancelledError): + await heur_task + return threat, _skipped_heuristic(final_url), True + heuristic = await heur_task + return threat, heuristic, False + + # heuristic 이 먼저 끝난 경로. threat 는 이미 거의 끝났을 가능성이 높으므로 대기한다. + heuristic = heur_task.result() + threat = await threat_task + # 양쪽 다 완료된 뒤에 malicious 로 드러난 경우는 4단계만 skip 하면 된다. + return threat, heuristic, threat.is_malicious + except BaseException: + # 상위 cancel 또는 stage 내부 예외 — 남은 task 가 고아로 떠돌지 않게 정리하고 전파. + for task in (threat_task, heur_task): + if not task.done(): + task.cancel() + raise + + async def run_pipeline( analysis_id: str, original_url: str, @@ -85,18 +229,55 @@ async def run_pipeline( ) unchain = await _stage_unchain(log, norm.normalized_url) - threat = await _stage_threat_db(log, unchain.final_url, session) - heuristic = await _stage_domain_heuristic(log, unchain.final_url) + # 2·3단계는 둘 다 unchain.final_url 만 필요하고 서로 독립이라 병렬로 돈다. + # threat_db 가 먼저 malicious 로 끝나면 verdict 가 이미 danger 로 확정이므로 + # heuristic 을 cancel 하고 4단계까지 skip — 여기서 조기 종료가 일어난다. + threat, heuristic, short_circuited = await _run_stage_2_and_3( + log, unchain.final_url, session + ) - log.info("pipeline.done", final_url=unchain.final_url) + if short_circuited: + log.info( + "pipeline.short_circuit", + reason="threat_db_match", + gsb_threat=threat.gsb.is_threat, + urlhaus_threat=threat.urlhaus.is_threat, + ) + content = skipped_already_danger(unchain.final_url) + else: + # 이미 danger 확정된 URL은 페이지를 받아보지 않는다 — 네트워크·AI 비용 절감. + # 판정이 바뀌지 않을 단계에 초 단위 지연과 건당 원화를 쓸 이유가 없다. + preceding = _preceding_score(threat, heuristic) + if threat.is_malicious or preceding >= settings.score_danger_threshold: + log.info( + "pipeline.content_analysis.skipped", + reason=("threat_db_match" if threat.is_malicious else "already_danger"), + preceding_score=preceding, + ) + content = skipped_already_danger(unchain.final_url) + else: + upstream = _collect_upstream_signals(threat, heuristic) + content = await _stage_content_analysis(log, unchain.final_url, upstream) + + score = _total_score(threat, heuristic, content) + verdict = _decide_verdict(score, threat) + log.info( + "pipeline.done", + final_url=unchain.final_url, + verdict=verdict.value, + score=score, + ) return PipelineSuccess( analysis_id=analysis_id, original_url=original_url, final_url=unchain.final_url, + verdict=verdict, + score=score, stages=PipelineStages( normalize=norm, unchain=unchain, threat_db=threat, domain_heuristic=heuristic, + content_analysis=content, ), ) From 7d8213dc49359ec4e8fae852e55a9a249058520c Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:40:18 +0900 Subject: [PATCH 086/173] =?UTF-8?q?[Feat]=20AI=20provider=20=EB=B6=80?= =?UTF-8?q?=ED=8A=B8=EC=8A=A4=ED=8A=B8=EB=9E=A9=20+=20dev=20=EB=8B=A8?= =?UTF-8?q?=EA=B3=84=EB=B3=84=205=EA=B0=9C=20=EC=97=94=EB=93=9C=ED=8F=AC?= =?UTF-8?q?=EC=9D=B8=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/v1/endpoints/dev.py | 205 ++++++++++++++++++++++++++++++++++-- app/main.py | 45 ++++++++ 2 files changed, 244 insertions(+), 6 deletions(-) diff --git a/app/api/v1/endpoints/dev.py b/app/api/v1/endpoints/dev.py index a4dd138..fdd2f11 100644 --- a/app/api/v1/endpoints/dev.py +++ b/app/api/v1/endpoints/dev.py @@ -1,33 +1,226 @@ -"""개발용 엔드포인트 — debug=true 환경에서만 등록됨.""" +"""개발용 엔드포인트 — debug=true 환경에서만 등록됨. + +각 파이프라인 단계를 단독으로 호출할 수 있는 5개 엔드포인트를 제공한다. +모든 엔드포인트는 raw URL 을 받아 normalize_url() 을 통과시킨 뒤 해당 단계만 실행한다 — +이 경로는 디버그/QA 용이며 운영 트래픽은 전부 /api/v1/analyze (비동기) 를 거친다. + +| Path | Stage | 호출 함수 | +|-------------------------------|---------------|---------------------------------| +| POST /dev/normalize | Stage 1 | normalize_url + unchain_url | +| POST /dev/threat-db | Stage 2 | check_threat_db | +| POST /dev/domain-heuristic | Stage 3 | check_domain_heuristic | +| POST /dev/content-analysis | Stage 4 | analyze_content | +| POST /dev/analyze | 전체 | run_pipeline | + +보조 디버그 엔드포인트(/dev/content/fetch-extract) 는 4단계 내부 fetch+extract 만 노출한다. +""" from __future__ import annotations import uuid -from fastapi import APIRouter -from pydantic import BaseModel +from fastapi import APIRouter, HTTPException, status +from pydantic import BaseModel, Field from app.api.deps import DBSession +from app.core.exceptions import NormalizationError +from app.schemas.content_analysis import ( + ContentAnalysisResult, + DevFetchExtractResponse, + ExtractedFeaturesView, + FetchStatusView, +) +from app.schemas.domain_heuristic import DomainHeuristicResult +from app.schemas.normalize import NormalizeResult from app.schemas.pipeline import PipelineFailure, PipelineSuccess +from app.schemas.threat_db import ThreatDbResult +from app.schemas.unchain import UnchainResult +from app.services.content_analyzer import analyze_content +from app.services.content_analyzer.extract import extract_features +from app.services.content_analyzer.fetch import fetch_page +from app.services.domain_heuristic import check_domain_heuristic +from app.services.normalizer import normalize_url from app.services.pipeline import run_pipeline +from app.services.threat_db import check_threat_db +from app.services.unchainer import unchain_url router = APIRouter() +# HTML 미리보기 상한 — 에디터/브라우저 렌더 부담 없이 원본을 훑기 좋은 크기. +# 슬라이싱은 codepoint 단위라 UTF-8 바이트 길이는 더 클 수 있다. +_HTML_PREVIEW_CHARS = 2048 + + +def _normalize_or_400(raw_url: str) -> str: + """dev 엔드포인트 공용 — 스킴/IPv6/포트 검증을 normalize_url 로 위임. + + 파이프라인은 1단계에서 이걸 거치지만 dev 엔드포인트는 raw URL 을 직접 받기 때문에 + 여기서 동일한 검증을 통과시켜 file:// 같은 비허용 스킴을 fetch 로 흘리지 않는다. + SSRF 의 사설 IP 차단은 fetch.py 1선에서 한 번 더 막는다. + """ + try: + return normalize_url(raw_url).normalized_url + except NormalizationError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"invalid url: {exc.message}", + ) from exc + + +class DevUrlRequest(BaseModel): + url: str = Field(description="raw URL — 내부에서 normalize_url 을 통과시킨다.") + class DevAnalyzeRequest(BaseModel): url: str +class DevContentRequest(BaseModel): + url: str + + +class DevNormalizeResponse(BaseModel): + """1단계 응답 — normalize 결과와 unchain 결과를 함께 반환.""" + + normalize: NormalizeResult + unchain: UnchainResult + + +# --------------------------------------------------------------------------- +# 전체 파이프라인 +# --------------------------------------------------------------------------- + + @router.post( "/analyze", response_model=PipelineSuccess | PipelineFailure, - summary="[Dev] URL 분석 — 동기 결과 반환", - description="파이프라인을 동기로 실행해 결과를 즉시 반환합니다. debug=true 환경 전용.", + summary="[Dev] 전체 파이프라인 — 동기 결과 반환", + description=( + "run_pipeline 을 동기로 실행해 1~4단계 + verdict 를 즉시 반환합니다. " + "debug=true 환경 전용." + ), ) -async def dev_analyze(body: DevAnalyzeRequest, session: DBSession) -> PipelineSuccess | PipelineFailure: +async def dev_analyze( + body: DevAnalyzeRequest, session: DBSession +) -> PipelineSuccess | PipelineFailure: analysis_id = str(uuid.uuid4()) return await run_pipeline( analysis_id=analysis_id, original_url=body.url, session=session, ) + + +# --------------------------------------------------------------------------- +# 단계별 API — Stage 1 ~ Stage 4 +# --------------------------------------------------------------------------- + + +@router.post( + "/normalize", + response_model=DevNormalizeResponse, + summary="[Dev] 1단계 — URL 정규화 + 단축 URL 언체이닝", + description=( + "normalize_url 로 canonical form 을 만든 뒤 unchain_url 로 리다이렉트 체인을 끝까지 따라가 " + "최종 URL 을 확정합니다. 2~4단계의 입력이 되는 final_url 을 별도로 확인할 때 사용합니다." + ), +) +async def dev_normalize(body: DevUrlRequest) -> DevNormalizeResponse: + try: + normalize_result = normalize_url(body.url) + except NormalizationError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"invalid url: {exc.message}", + ) from exc + unchain_result = await unchain_url(normalize_result.normalized_url) + return DevNormalizeResponse(normalize=normalize_result, unchain=unchain_result) + + +@router.post( + "/threat-db", + response_model=ThreatDbResult, + summary="[Dev] 2단계 — 외부 위협 DB 대조 (GSB + URLhaus)", + description=( + "check_threat_db 를 직접 호출합니다. 입력 URL 은 normalize_url 만 거친 상태로 전달되며 " + "단축 URL 을 풀고 싶다면 먼저 /dev/normalize 로 final_url 을 얻어 그 값을 넣어야 합니다." + ), +) +async def dev_threat_db(body: DevUrlRequest, session: DBSession) -> ThreatDbResult: + target = _normalize_or_400(body.url) + return await check_threat_db(session, target) + + +@router.post( + "/domain-heuristic", + response_model=DomainHeuristicResult, + summary="[Dev] 3단계 — 도메인 휴리스틱 분석", + description=( + "check_domain_heuristic 를 직접 호출합니다. RDAP 캐시(24h) 가 적용되며 캐시 미스 시 " + "최대 약 5초 대기할 수 있습니다. 단축 URL 을 풀고 싶다면 /dev/normalize 를 먼저 호출하세요." + ), +) +async def dev_domain_heuristic(body: DevUrlRequest) -> DomainHeuristicResult: + target = _normalize_or_400(body.url) + return await check_domain_heuristic(target) + + +@router.post( + "/content-analysis", + response_model=ContentAnalysisResult, + summary="[Dev] 4단계 — 페이지 콘텐츠 정적 분석 (fetch + extract + signals + AI)", + description=( + "analyze_content 를 직접 호출해 규칙 점수 · AI 추론이 합쳐진 ContentAnalysisResult 를 " + "그대로 반환합니다. 파이프라인 skip 로직은 거치지 않으므로 선행 단계 점수와 무관하게 항상 " + "실제 분석을 수행합니다. AI 프로바이더는 AI_PROVIDER 설정(기본 auto)을 따릅니다. " + "이 엔드포인트는 unchainer 를 거치지 않으므로 단축 URL 을 직접 넣으면 " + "unexpected_redirect 로 떨어집니다 — 단축 URL 은 /dev/normalize 로 먼저 풀어야 합니다. " + "선행 단계 신호 없이 호출되므로 AI 프롬프트는 페이지 피처만 보고 판정합니다." + ), +) +async def dev_content_analysis(body: DevUrlRequest) -> ContentAnalysisResult: + return await analyze_content(_normalize_or_400(body.url)) + + +# --------------------------------------------------------------------------- +# 보조 디버그 — 4단계 내부 fetch+extract 만 분리해서 확인 +# --------------------------------------------------------------------------- + + +@router.post( + "/content/fetch-extract", + response_model=DevFetchExtractResponse, + summary="[Dev] (보조) 크롤링 + HTML 파싱 결과 확인", + description=( + "fetch_page() + extract_features() 만 수행해서 원본 HTML preview 와 추출된 " + "피처(title, password 유무, meta refresh, 외부 링크 비율, 이미지 alt)를 반환합니다. " + "규칙 점수 · AI 추론은 돌리지 않습니다. 4단계 디버깅용 보조 엔드포인트." + ), +) +async def dev_content_fetch_extract(body: DevContentRequest) -> DevFetchExtractResponse: + target = _normalize_or_400(body.url) + fetch_result = await fetch_page(target) + fetch_view = FetchStatusView( + ok=fetch_result.ok, + status_code=fetch_result.status_code, + html_length=len(fetch_result.html), + error=fetch_result.error, + ) + + if not fetch_result.ok: + return DevFetchExtractResponse(url=target, fetch=fetch_view) + + features = extract_features(fetch_result.html, base_url=target) + return DevFetchExtractResponse( + url=target, + fetch=fetch_view, + features=ExtractedFeaturesView( + title=features.title, + has_password_field=features.has_password_field, + has_meta_refresh=features.has_meta_refresh, + external_link_ratio=features.external_link_ratio, + image_alts=features.image_alts, + is_spa_shell=features.is_spa_shell, + ), + html_preview=fetch_result.html[:_HTML_PREVIEW_CHARS], + ) diff --git a/app/main.py b/app/main.py index b67c983..26a0d96 100644 --- a/app/main.py +++ b/app/main.py @@ -12,11 +12,44 @@ from app.core.scheduler import shutdown_scheduler, start_scheduler from app.db.session import engine from app.middleware.request_context import RequestContextMiddleware +from app.services.content_analyzer.ai import AIProvider, NullAIProvider, set_ai_provider +from app.services.content_analyzer.ai_openai import OpenAIProvider +from app.services.content_analyzer.fetch import aclose_client as aclose_fetch_client from app.services.domain_heuristic.rdap import aclose_client as aclose_rdap_client logger = get_logger(__name__) +def _select_ai_provider() -> AIProvider: + """settings.ai_provider + 키 존재 여부로 프로바이더를 하나 고른다. + + - "null" : 항상 NullAIProvider (정상 비활성) + - "openai" : 키 누락이면 경고 로그 + NullAIProvider(fallback_reason="provider_misconfigured") + - "auto" : 키 있으면 OpenAIProvider, 없으면 NullAIProvider (정상 비활성) + """ + choice = settings.ai_provider + if choice == "null": + return NullAIProvider() + if choice == "openai": + if settings.openai_api_key: + return OpenAIProvider() + # "강제 openai" 인데 키가 없는 misconfiguration — 응답에 fallback 흔적을 남겨 + # 정상 NullProvider 동작과 운영자가 구분할 수 있게 한다. + logger.warning("app.ai_provider.missing_key", provider="openai") + return NullAIProvider(fallback_reason="provider_misconfigured") + # auto + if settings.openai_api_key: + return OpenAIProvider() + return NullAIProvider() + + +async def _aclose_provider(provider: AIProvider) -> None: + # AIProvider Protocol 에는 aclose 가 없다 — 구현체에만 있으면 호출. + aclose = getattr(provider, "aclose", None) + if aclose is not None: + await aclose() + + @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: configure_logging() @@ -26,6 +59,16 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: version=settings.app_version, ) start_scheduler() + + # AI 프로바이더 부트스트랩 — 키가 없으면 NullAIProvider (4단계 AI 추론 비활성). + # 로컬/CI 에서도 이 분기로 안전하게 동작하므로 별도 조건이 필요 없다. + ai_provider = _select_ai_provider() + set_ai_provider(ai_provider) + logger.info( + "app.ai_provider", + provider=type(ai_provider).__name__, + choice=settings.ai_provider, + ) startup_task: asyncio.Task | None = None if settings.scheduler_enabled and settings.urlhaus_sync_on_startup: # 최초 부트 시 즉시 1회 동기화 (백그라운드, 앱 기동은 블로킹하지 않음). @@ -61,6 +104,8 @@ def _log_startup_sync_result(t: asyncio.Task) -> None: ) shutdown_scheduler(wait=False) await aclose_rdap_client() + await aclose_fetch_client() + await _aclose_provider(ai_provider) await engine.dispose() From ecb07d466f40ab19875ff663da9c3a6d12530572 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:41:06 +0900 Subject: [PATCH 087/173] =?UTF-8?q?[Test]=204=EB=8B=A8=EA=B3=84=20?= =?UTF-8?q?=EB=B6=84=EC=84=9D=EA=B8=B0=20=C2=B7=20dev=20=EC=97=94=EB=93=9C?= =?UTF-8?q?=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=C2=B7=20=ED=8C=8C=EC=9D=B4?= =?UTF-8?q?=ED=94=84=EB=9D=BC=EC=9D=B8=20=EB=B3=91=EB=A0=AC/short-circuit?= =?UTF-8?q?=20=ED=9A=8C=EA=B7=80=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/api/test_dev_content.py | 147 ++++++ tests/services/content_analyzer/__init__.py | 0 tests/services/content_analyzer/test_ai.py | 58 +++ .../content_analyzer/test_ai_openai.py | 251 ++++++++++ .../services/content_analyzer/test_analyze.py | 318 ++++++++++++ .../services/content_analyzer/test_extract.py | 218 ++++++++ tests/services/content_analyzer/test_fetch.py | 337 +++++++++++++ .../services/content_analyzer/test_signals.py | 182 +++++++ tests/services/test_pipeline.py | 467 +++++++++++++++++- 9 files changed, 1975 insertions(+), 3 deletions(-) create mode 100644 tests/api/test_dev_content.py create mode 100644 tests/services/content_analyzer/__init__.py create mode 100644 tests/services/content_analyzer/test_ai.py create mode 100644 tests/services/content_analyzer/test_ai_openai.py create mode 100644 tests/services/content_analyzer/test_analyze.py create mode 100644 tests/services/content_analyzer/test_extract.py create mode 100644 tests/services/content_analyzer/test_fetch.py create mode 100644 tests/services/content_analyzer/test_signals.py diff --git a/tests/api/test_dev_content.py b/tests/api/test_dev_content.py new file mode 100644 index 0000000..265b1b1 --- /dev/null +++ b/tests/api/test_dev_content.py @@ -0,0 +1,147 @@ +"""/api/v1/dev/content/* — 크롤링 결과·콘텐츠 분석 디버그 엔드포인트.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from unittest.mock import AsyncMock, patch + +import httpx +import pytest_asyncio +from app.api.v1.endpoints import dev as dev_endpoint +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict, ContentAnalysisResult, ContentSignal +from app.services.content_analyzer.fetch import FetchResult +from fastapi import FastAPI +from httpx import ASGITransport + + +@pytest_asyncio.fixture +async def client() -> AsyncIterator[httpx.AsyncClient]: + """dev 라우터만 마운트한 얇은 FastAPI 앱 — lifespan / DB / 스케줄러를 건드리지 않는다.""" + app = FastAPI() + app.include_router(dev_endpoint.router, prefix=f"{settings.api_v1_prefix}/dev") + transport = ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +_ENDPOINT_FETCH = f"{settings.api_v1_prefix}/dev/content/fetch-extract" +_ENDPOINT_ANALYSIS = f"{settings.api_v1_prefix}/dev/content-analysis" + + +class TestFetchExtractEndpoint: + async def test_returns_fetch_and_features_on_success(self, client: httpx.AsyncClient) -> None: + html = ( + "NAVER" + '
' + 'NAVER logo' + ) + fetch = FetchResult( + ok=True, + url="https://evil-naver.test/", + status_code=200, + html=html, + ) + with patch( + "app.api.v1.endpoints.dev.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://evil-naver.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["url"] == "https://evil-naver.test/" + assert body["fetch"]["ok"] is True + assert body["fetch"]["status_code"] == 200 + assert body["fetch"]["html_length"] == len(html) + assert body["features"]["title"] == "NAVER" + assert body["features"]["has_password_field"] is True + assert body["features"]["image_alts"] == ["NAVER logo"] + assert body["html_preview"] is not None + assert "NAVER" in body["html_preview"] + + async def test_fetch_failure_omits_features(self, client: httpx.AsyncClient) -> None: + fetch = FetchResult(ok=False, url="https://down.test/", error="connect_error") + with patch( + "app.api.v1.endpoints.dev.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://down.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["fetch"]["ok"] is False + assert body["fetch"]["error"] == "connect_error" + assert body["features"] is None + assert body["html_preview"] is None + + async def test_html_preview_caps_bytes(self, client: httpx.AsyncClient) -> None: + big = "" + ("x" * 10000) + "" + fetch = FetchResult(ok=True, url="https://big.test/", status_code=200, html=big) + with patch( + "app.api.v1.endpoints.dev.fetch_page", + AsyncMock(return_value=fetch), + ): + resp = await client.post(_ENDPOINT_FETCH, json={"url": "https://big.test/"}) + + preview = resp.json()["html_preview"] + assert preview is not None + # 2048 bytes 상한 — 그 이하로 잘려야 한다 + assert len(preview) <= 2048 + + async def test_missing_url_returns_422(self, client: httpx.AsyncClient) -> None: + resp = await client.post(_ENDPOINT_FETCH, json={}) + assert resp.status_code == 422 + + async def test_invalid_scheme_rejected(self, client: httpx.AsyncClient) -> None: + """file:// 같은 비허용 스킴은 normalize_url 에서 400 으로 떨어진다.""" + resp = await client.post(_ENDPOINT_FETCH, json={"url": "file:///etc/passwd"}) + assert resp.status_code == 400 + assert "invalid url" in resp.json()["detail"] + + +class TestContentAnalysisEndpoint: + async def test_returns_full_content_analysis(self, client: httpx.AsyncClient) -> None: + expected = ContentAnalysisResult( + final_url="https://x.test/", + fetched=True, + score=80, + signals=[ContentSignal.BRAND_IMPERSONATION_FORM], + title="NAVER", + has_password_field=True, + brand_impersonation=True, + ai_verdict=AIVerdict.PHISHING, + ai_reason="브랜드 위장", + ) + with patch( + "app.api.v1.endpoints.dev.analyze_content", + AsyncMock(return_value=expected), + ): + resp = await client.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + + assert resp.status_code == 200 + body = resp.json() + assert body["final_url"] == "https://x.test/" + assert body["score"] == 80 + assert body["signals"] == ["BRAND_IMPERSONATION_FORM"] + assert body["ai_verdict"] == "phishing" + assert body["ai_reason"] == "브랜드 위장" + + async def test_does_not_skip_based_on_preceding_score( + self, client: httpx.AsyncClient + ) -> None: + """파이프라인과 달리 dev 엔드포인트는 항상 실제 분석을 수행한다.""" + expected = ContentAnalysisResult( + final_url="https://x.test/", + fetched=False, + score=0, + signals=[], + error="connect_error", + ) + with patch( + "app.api.v1.endpoints.dev.analyze_content", + AsyncMock(return_value=expected), + ) as mock_analyze: + await client.post(_ENDPOINT_ANALYSIS, json={"url": "https://x.test/"}) + + mock_analyze.assert_awaited_once_with("https://x.test/") diff --git a/tests/services/content_analyzer/__init__.py b/tests/services/content_analyzer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/services/content_analyzer/test_ai.py b/tests/services/content_analyzer/test_ai.py new file mode 100644 index 0000000..5918a77 --- /dev/null +++ b/tests/services/content_analyzer/test_ai.py @@ -0,0 +1,58 @@ +"""ai.py — 프로바이더 Protocol + Null 기본 어댑터.""" + +from __future__ import annotations + +import pytest +from app.schemas.content_analysis import AIVerdict +from app.services.content_analyzer.ai import ( + AIInference, + AIPromptContext, + NullAIProvider, + get_ai_provider, + set_ai_provider, +) + + +@pytest.fixture(autouse=True) +def _reset_provider() -> None: + """각 테스트 종료 시 기본 NullAIProvider 로 복구.""" + yield + set_ai_provider(NullAIProvider()) + + +async def test_null_provider_returns_none() -> None: + ctx = AIPromptContext( + final_url="https://x.test/", + title="hi", + has_password_field=False, + has_meta_refresh=False, + image_alts=(), + external_link_ratio=None, + ) + result = await NullAIProvider().infer(ctx) + assert result is None + + +async def test_default_provider_is_null() -> None: + assert isinstance(get_ai_provider(), NullAIProvider) + + +async def test_set_custom_provider() -> None: + class FakeProvider: + async def infer(self, ctx: AIPromptContext) -> AIInference | None: + return AIInference(verdict=AIVerdict.PHISHING, reason="mock") + + set_ai_provider(FakeProvider()) + provider = get_ai_provider() + ctx = AIPromptContext( + final_url="https://x.test/", + title=None, + has_password_field=False, + has_meta_refresh=False, + image_alts=(), + external_link_ratio=None, + ) + inference = await provider.infer(ctx) + assert inference is not None + assert inference.verdict == AIVerdict.PHISHING + assert inference.reason == "mock" diff --git a/tests/services/content_analyzer/test_ai_openai.py b/tests/services/content_analyzer/test_ai_openai.py new file mode 100644 index 0000000..d2b8413 --- /dev/null +++ b/tests/services/content_analyzer/test_ai_openai.py @@ -0,0 +1,251 @@ +"""OpenAIProvider — gpt-4o-mini 기반 피싱 추론 어댑터.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict +from app.services.content_analyzer.ai import AIPromptContext +from app.services.content_analyzer.ai_openai import OpenAIProvider + + +@pytest.fixture(autouse=True) +def _set_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_api_key", "test-key") + + +def _ctx() -> AIPromptContext: + return AIPromptContext( + final_url="https://evil-naver.test/signin", + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=False, + image_alts=("NAVER",), + external_link_ratio=0.9, + ) + + +def _mock_completion( + content: str, + *, + usage: SimpleNamespace | None = None, +) -> SimpleNamespace: + """chat.completions.create 가 돌려주는 객체 모양을 흉내낸 간단 스텁.""" + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=content))], + usage=usage, + ) + + +def _usage(prompt: int, completion: int) -> SimpleNamespace: + return SimpleNamespace( + prompt_tokens=prompt, + completion_tokens=completion, + total_tokens=prompt + completion, + ) + + +async def test_openai_returns_verdict_on_success() -> None: + payload = json.dumps({"verdict": "phishing", "reason": "브랜드 불일치"}) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=AsyncMock( + return_value=_mock_completion(payload, usage=_usage(120, 18)) + ) + ) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + provider = OpenAIProvider() + result = await provider.infer(_ctx()) + + assert result is not None + assert result.verdict == AIVerdict.PHISHING + assert result.reason == "브랜드 불일치" + # 모델 id 와 토큰 사용량이 추론 결과에 실려 나와야 한다 + assert result.model == settings.openai_model + assert result.token_usage is not None + assert result.token_usage.prompt_tokens == 120 + assert result.token_usage.completion_tokens == 18 + assert result.token_usage.total_tokens == 138 + + +async def test_openai_missing_usage_returns_none_token_usage() -> None: + """usage 가 비어 있으면 token_usage=None 으로 떨어뜨린다 (추론 자체는 성공).""" + payload = json.dumps({"verdict": "benign", "reason": "ok"}) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=AsyncMock(return_value=_mock_completion(payload, usage=None)) + ) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + result = await OpenAIProvider().infer(_ctx()) + + assert result is not None + assert result.token_usage is None + + +async def test_openai_custom_model_overrides_settings() -> None: + """생성자에 model 을 넘기면 settings.openai_model 보다 우선한다 — 모델 비교용.""" + payload = json.dumps({"verdict": "benign", "reason": "ok"}) + create = AsyncMock(return_value=_mock_completion(payload, usage=_usage(10, 5))) + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + result = await OpenAIProvider(model="gpt-4o").infer(_ctx()) + + assert create.call_args.kwargs["model"] == "gpt-4o" + assert result is not None + assert result.model == "gpt-4o" + + +async def test_openai_returns_none_without_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_api_key", None) + provider = OpenAIProvider() + assert await provider.infer(_ctx()) is None + + +async def test_openai_uses_configured_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "openai_model", "gpt-4o-mini") + + create = AsyncMock( + return_value=_mock_completion(json.dumps({"verdict": "benign", "reason": "ok"})) + ) + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + await OpenAIProvider().infer(_ctx()) + + kwargs = create.call_args.kwargs + assert kwargs["model"] == "gpt-4o-mini" + # structured output 스키마 사용 + assert kwargs["response_format"]["type"] == "json_schema" + assert kwargs["response_format"]["json_schema"]["strict"] is True + # timeout 전달 + assert kwargs["timeout"] == settings.openai_timeout_seconds + + +async def test_openai_user_prompt_includes_upstream_signals() -> None: + """ctx.upstream_signals 가 user payload 의 upstream_signals 배열로 직렬화돼야 한다.""" + payload = json.dumps({"verdict": "phishing", "reason": "타이포 + 비밀번호 폼"}) + create = AsyncMock(return_value=_mock_completion(payload, usage=_usage(50, 10))) + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + ctx = AIPromptContext( + final_url="https://evil-naverr.test/signin", + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=False, + image_alts=("NAVER",), + external_link_ratio=0.9, + upstream_signals=("TYPO_DOMAIN", "NEW_DOMAIN"), + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + await OpenAIProvider().infer(ctx) + + user_msg = create.call_args.kwargs["messages"][1]["content"] + body = json.loads(user_msg) + assert body["upstream_signals"] == ["TYPO_DOMAIN", "NEW_DOMAIN"] + + +async def test_openai_user_prompt_omits_empty_upstream_signals() -> None: + """기본값(빈 튜플)이면 빈 배열로 직렬화돼 단독 페이지 분석 모드와 동일하게 동작.""" + payload = json.dumps({"verdict": "benign", "reason": "ok"}) + create = AsyncMock(return_value=_mock_completion(payload)) + client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=create)) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + await OpenAIProvider().infer(_ctx()) + + body = json.loads(create.call_args.kwargs["messages"][1]["content"]) + assert body["upstream_signals"] == [] + + +async def test_openai_parse_error_returns_none() -> None: + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=AsyncMock(return_value=_mock_completion("not-json")) + ) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + result = await OpenAIProvider().infer(_ctx()) + assert result is None + + +async def test_openai_unknown_verdict_returns_none() -> None: + """응답 JSON 의 verdict 값이 enum 범위를 벗어나면 None 으로 떨어뜨린다.""" + payload = json.dumps({"verdict": "unknown_value", "reason": "x"}) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace(create=AsyncMock(return_value=_mock_completion(payload))) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + assert await OpenAIProvider().infer(_ctx()) is None + + +async def test_openai_api_exception_returns_none() -> None: + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace(create=AsyncMock(side_effect=RuntimeError("5xx"))) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ): + result = await OpenAIProvider().infer(_ctx()) + assert result is None + + +async def test_openai_cancelled_propagates() -> None: + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=AsyncMock(side_effect=asyncio.CancelledError()) + ) + ) + ) + with patch( + "app.services.content_analyzer.ai_openai.AsyncOpenAI", + return_value=client, + ), pytest.raises(asyncio.CancelledError): + await OpenAIProvider().infer(_ctx()) diff --git a/tests/services/content_analyzer/test_analyze.py b/tests/services/content_analyzer/test_analyze.py new file mode 100644 index 0000000..64bf77f --- /dev/null +++ b/tests/services/content_analyzer/test_analyze.py @@ -0,0 +1,318 @@ +"""analyze_content — fetch · extract · signals · AI 통합.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from app.core.config import settings +from app.schemas.content_analysis import AIVerdict, ContentSignal, TokenUsage +from app.services.content_analyzer.ai import AIInference, AIPromptContext, NullAIProvider +from app.services.content_analyzer.analyze import analyze_content +from app.services.content_analyzer.fetch import FetchResult + + +def _mock_fetch(ok: bool, html: str = "", error: str | None = None, status: int = 200): + return patch( + "app.services.content_analyzer.analyze.fetch_page", + AsyncMock( + return_value=FetchResult( + ok=ok, + url="https://x.test/", + status_code=status if ok else None, + html=html, + error=error, + ) + ), + ) + + +class TestHappyPath: + async def test_clean_page_returns_zero_score(self) -> None: + html = "Hello

ok

" + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://benign.test/") + + assert result.fetched is True + assert result.final_url == "https://benign.test/" + assert result.score == 0 + assert result.signals == [] + assert result.title == "Hello" + assert result.has_password_field is False + assert result.error is None + + +class TestFetchFailure: + async def test_fetch_failure_degraded_result(self) -> None: + with _mock_fetch(ok=False, error="timeout"): + result = await analyze_content("https://down.test/") + + assert result.fetched is False + assert result.error == "timeout" + assert ContentSignal.FETCH_FAILED in result.signals + assert result.score == settings.score_weight_content_fetch_failed + assert result.ai_verdict is None + + @pytest.mark.parametrize("error", ["not_html", "too_large", "unexpected_redirect"]) + async def test_benign_fetch_errors_score_zero(self, error: str) -> None: + """정상 컨텐츠(PDF/이미지/대형 정적)·파이프라인 이슈는 시그널만 남기고 가산은 0.""" + with _mock_fetch(ok=False, error=error): + result = await analyze_content("https://x.test/") + + assert result.score == 0 + assert ContentSignal.FETCH_FAILED in result.signals + assert result.error == error + + async def test_fetch_failure_skips_extract_and_ai(self) -> None: + with ( + _mock_fetch(ok=False, error="connect_error"), + patch("app.services.content_analyzer.analyze.extract_features") as ext_mock, + patch("app.services.content_analyzer.analyze.score_content") as sig_mock, + patch("app.services.content_analyzer.analyze.get_ai_provider") as ai_mock, + ): + await analyze_content("https://x.test/") + + ext_mock.assert_not_called() + sig_mock.assert_not_called() + ai_mock.assert_not_called() + + +class TestBrandImpersonationEndToEnd: + async def test_brand_impersonation_form_scored(self) -> None: + html = """ + + NAVER 로그인 + +
+ NAVER + + + """ + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://evil-naver.test/signin") + + assert result.fetched is True + assert result.brand_impersonation is True + assert result.logo_alt_impersonation is True + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.title == "NAVER 로그인" + assert result.has_password_field is True + + +class TestAI: + async def test_ai_phishing_verdict_adds_score(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.PHISHING, reason="brand mismatch") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + + html = "hi" + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.PHISHING + assert result.ai_reason == "brand mismatch" + assert result.score >= settings.score_weight_ai_phishing + + async def test_ai_suspicious_verdict(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="mild") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.SUSPICIOUS + assert result.score == settings.score_weight_ai_suspicious + + async def test_ai_benign_verdict_no_score(self, monkeypatch: pytest.MonkeyPatch) -> None: + class StubAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference(verdict=AIVerdict.BENIGN, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: StubAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict == AIVerdict.BENIGN + assert result.score == 0 + + async def test_ai_none_keeps_nulls(self) -> None: + # 기본 provider 는 NullAIProvider → None 반환 + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict is None + assert result.ai_reason is None + assert result.ai_error is None + assert result.ai_model is None + assert result.ai_token_usage is None + + async def test_ai_token_usage_and_model_surface( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """프로바이더가 채워 준 model/token_usage 가 결과에 그대로 실려 나와야 한다.""" + + class TokenAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + return AIInference( + verdict=AIVerdict.BENIGN, + reason="ok", + model="gpt-4o-mini", + token_usage=TokenUsage( + prompt_tokens=100, + completion_tokens=20, + total_tokens=120, + ), + ) + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: TokenAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_model == "gpt-4o-mini" + assert result.ai_token_usage is not None + assert result.ai_token_usage.total_tokens == 120 + + async def test_ai_exception_is_absorbed(self, monkeypatch: pytest.MonkeyPatch) -> None: + class BoomAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + raise RuntimeError("model exploded") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: BoomAI(), + ) + with _mock_fetch(ok=True, html=""): + result = await analyze_content("https://x.test/") + + assert result.ai_verdict is None + assert result.ai_error is not None + # 규칙 기반 결과는 여전히 유효 + assert result.fetched is True + + +class TestSpaShellEndToEnd: + async def test_spa_shell_flag_surfaces_and_ai_sees_it( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """SPA 셸 HTML 에 대해 ContentAnalysisResult.is_spa_shell=True 로 나오고, + AI 프로바이더의 프롬프트 컨텍스트에도 is_spa_shell=True 가 실려 전달된다.""" + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.BENIGN, reason="spa shell, no evidence") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + html = ( + 'x' + '' + '
' + ) + with _mock_fetch(ok=True, html=html): + result = await analyze_content("https://spa.test/") + + assert result.is_spa_shell is True + assert result.has_password_field is False + assert ContentSignal.SPA_SHELL in result.signals + # SPA 셸은 규칙 점수 가산 없음 — AI benign 이므로 최종 점수 0 + assert result.score == 0 + assert len(seen) == 1 + assert seen[0].is_spa_shell is True + + +class TestCancelledError: + async def test_fetch_cancelled_propagates(self) -> None: + """shutdown/timeout 신호는 degraded 결과로 흡수하지 않고 re-raise.""" + with patch( + "app.services.content_analyzer.analyze.fetch_page", + AsyncMock(side_effect=asyncio.CancelledError()), + ), pytest.raises(asyncio.CancelledError): + await analyze_content("https://x.test/") + + async def test_ai_cancelled_propagates(self, monkeypatch: pytest.MonkeyPatch) -> None: + class CancelAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + raise asyncio.CancelledError() + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CancelAI(), + ) + with _mock_fetch(ok=True, html=""), pytest.raises(asyncio.CancelledError): + await analyze_content("https://x.test/") + + +async def test_null_provider_used_by_default() -> None: + """설정 없이 호출 시 NullAIProvider 가 쓰이고 AI 필드는 None.""" + from app.services.content_analyzer import ai as ai_module + + assert isinstance(ai_module.get_ai_provider(), NullAIProvider) + + +class TestUpstreamSignals: + """analyze_content 가 받은 upstream_signals 를 AI 프롬프트 컨텍스트로 그대로 전달해야 한다.""" + + async def test_upstream_signals_forwarded_to_provider( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.SUSPICIOUS, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + with _mock_fetch(ok=True, html="x"): + await analyze_content( + "https://x.test/", + upstream_signals=["TYPO_DOMAIN", "NEW_DOMAIN"], + ) + + assert len(seen) == 1 + assert seen[0].upstream_signals == ("TYPO_DOMAIN", "NEW_DOMAIN") + + async def test_upstream_signals_default_empty( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """파라미터를 생략하면 빈 튜플 — 디버그 엔드포인트처럼 선행 단계 없이 호출되는 경로 보호.""" + seen: list[AIPromptContext] = [] + + class CaptureAI: + async def infer(self, ctx: AIPromptContext) -> AIInference: + seen.append(ctx) + return AIInference(verdict=AIVerdict.BENIGN, reason="ok") + + monkeypatch.setattr( + "app.services.content_analyzer.analyze.get_ai_provider", + lambda: CaptureAI(), + ) + with _mock_fetch(ok=True, html=""): + await analyze_content("https://x.test/") + + assert seen[0].upstream_signals == () diff --git a/tests/services/content_analyzer/test_extract.py b/tests/services/content_analyzer/test_extract.py new file mode 100644 index 0000000..4259bcd --- /dev/null +++ b/tests/services/content_analyzer/test_extract.py @@ -0,0 +1,218 @@ +"""extract_features — BS4로 title, password, meta refresh, img alt, 외부 링크 비율 추출.""" + +from __future__ import annotations + +import pytest +from app.services.content_analyzer.extract import extract_features + + +class TestTitle: + def test_title_plain(self) -> None: + html = "NAVER 로그인" + features = extract_features(html, base_url="https://evil.test/") + assert features.title == "NAVER 로그인" + + def test_title_trimmed(self) -> None: + html = " Kakao " + features = extract_features(html, base_url="https://x.test/") + assert features.title == "Kakao" + + def test_no_title(self) -> None: + html = "hi" + features = extract_features(html, base_url="https://x.test/") + assert features.title is None + + +class TestPasswordField: + def test_has_password(self) -> None: + html = '
' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is True + + def test_no_password(self) -> None: + html = '
' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is False + + def test_password_type_case_insensitive(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_password_field is True + + +class TestMetaRefresh: + def test_has_meta_refresh(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is True + + def test_no_meta_refresh(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is False + + def test_meta_refresh_case_insensitive(self) -> None: + html = '' + features = extract_features(html, base_url="https://x.test/") + assert features.has_meta_refresh is True + + +class TestImageAlts: + def test_collect_alts(self) -> None: + html = 'NAVERGoogle' + features = extract_features(html, base_url="https://x.test/") + assert features.image_alts == ["NAVER", "Google"] + + def test_empty_alts_ignored(self) -> None: + html = '   ' + features = extract_features(html, base_url="https://x.test/") + assert features.image_alts == [] + + def test_alts_capped_at_extraction(self) -> None: + """악성 페이지가 alt 수만 개를 박아 시그널 비용을 키우지 못하게 200 상한.""" + html = "".join(f'alt-{i}' for i in range(1000)) + features = extract_features(html, base_url="https://x.test/") + assert len(features.image_alts) == 200 + + +class TestExternalLinkRatio: + def test_all_internal(self) -> None: + html = """ + A + B + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 0.0 + + def test_all_external(self) -> None: + html = """ + F + B + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 1.0 + + def test_mixed(self) -> None: + html = """ + A + F + B + X + """ + features = extract_features(html, base_url="https://example.test/") + # 4개 중 2개 외부 + assert features.external_link_ratio == 0.5 + + def test_no_anchors_returns_none(self) -> None: + html = "

no links

" + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio is None + + def test_ignore_non_http_schemes(self) -> None: + """javascript:/mailto:/tel: 등은 네비게이션 링크가 아니므로 분모에서 제외.""" + html = """ + J + M + T + F + """ + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 1.0 + + def test_relative_resolved_against_base(self) -> None: + html = 'S' + features = extract_features(html, base_url="https://example.test/") + assert features.external_link_ratio == 0.0 + + +class TestSpaShellDetection: + def test_react_vite_shell_detected(self) -> None: + html = ( + 'x' + '' + '
' + ) + features = extract_features(html, base_url="https://x.test/") + assert features.is_spa_shell is True + assert features.has_password_field is False # 판정 불가 상태 + + def test_vue_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_nextjs_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_nuxt_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_svelte_shell_detected(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_angular_tag_shell_detected(self) -> None: + html = "" + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + def test_ssr_page_with_form_not_flagged(self) -> None: + """SSR 결과로 이미 form 이 존재하면 id=root 가 있어도 판정 가능 상태 — 셸 아님.""" + html = ( + '
' + '
' + "
" + ) + features = extract_features(html, base_url="https://x.test/") + assert features.is_spa_shell is False + assert features.has_password_field is True + + def test_static_page_without_mount_point_not_flagged(self) -> None: + """마운트 컨테이너도 form 도 없는 일반 정적 페이지는 셸 아님 — 그냥 콘텐츠 없는 페이지.""" + html = "

just text

" + assert extract_features(html, base_url="https://x.test/").is_spa_shell is False + + def test_input_without_form_still_disqualifies(self) -> None: + """form 없이 input 만 있어도 '판정 불가' 는 아니다 — 정적 추출이 이미 결정적이다.""" + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is False + + def test_mount_id_case_insensitive(self) -> None: + html = '
' + assert extract_features(html, base_url="https://x.test/").is_spa_shell is True + + +class TestInvalidHtml: + def test_empty_html_does_not_raise(self) -> None: + features = extract_features("", base_url="https://x.test/") + assert features.title is None + assert features.has_password_field is False + assert features.has_meta_refresh is False + assert features.external_link_ratio is None + assert features.image_alts == [] + + def test_malformed_html_recovers(self) -> None: + """BS4 html.parser는 깨진 HTML도 best-effort 파싱 — 예외 없이 반환.""" + html = "X" + features = extract_features(html, base_url="https://x.test/") + assert features.title == "X" + assert features.has_password_field is True + + +@pytest.mark.parametrize( + "href,expected_external", + [ + ("/path", False), + ("https://example.test/x", False), + ("https://www.example.test/x", False), # www. 서브도메인은 같은 사이트로 취급 + ("https://other.test/", True), + ("//other.test/", True), # protocol-relative + ], +) +def test_external_classification(href: str, expected_external: bool) -> None: + html = f'L' + features = extract_features(html, base_url="https://example.test/") + if expected_external: + assert features.external_link_ratio == 1.0 + else: + assert features.external_link_ratio == 0.0 diff --git a/tests/services/content_analyzer/test_fetch.py b/tests/services/content_analyzer/test_fetch.py new file mode 100644 index 0000000..10b1327 --- /dev/null +++ b/tests/services/content_analyzer/test_fetch.py @@ -0,0 +1,337 @@ +"""fetch_page — httpx 기반 페이지 본문 취득.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from app.core.config import settings +from app.services.content_analyzer import fetch as fetch_module +from app.services.content_analyzer.fetch import fetch_page + + +@pytest.fixture(autouse=True) +def _reset_singleton_client(monkeypatch: pytest.MonkeyPatch) -> None: + """매 테스트마다 모듈 싱글턴 클라이언트를 리셋 — 테스트 간 mock 누수 방지.""" + fetch_module._client = None + # SSRF DNS 2선이 unit 테스트에서 실제 외부 해석을 시도하면 느려지고 비결정적이 된다. + # 명시적으로 false 반환하도록 패치하고, 해당 동작은 별도 케이스에서 검증한다. + + async def _no_block(_host: str) -> bool: + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", _no_block + ) + yield + fetch_module._client = None + + +def _make_stream_response( + status_code: int = 200, + headers: dict[str, str] | None = None, + body: bytes = b"", +) -> AsyncMock: + """httpx.Response 스트림 mock — aiter_bytes + headers + status_code.""" + resp = AsyncMock() + resp.status_code = status_code + resp.headers = headers or {"content-type": "text/html; charset=utf-8"} + resp.charset_encoding = "utf-8" + + async def _aiter_bytes() -> object: + yield body + + resp.aiter_bytes = _aiter_bytes + resp.aclose = AsyncMock() + resp.aread = AsyncMock(return_value=body) + return resp + + +def _patch_client(send_return: object | None = None, *, send_side_effect: object = None): + """모듈 싱글턴 _get_client 를 mock 으로 갈아끼운다 — send/build_request 만 노출.""" + mock_client = AsyncMock() + mock_client.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + if send_side_effect is not None: + mock_client.send = AsyncMock(side_effect=send_side_effect) + else: + mock_client.send = AsyncMock(return_value=send_return) + return patch( + "app.services.content_analyzer.fetch._get_client", return_value=mock_client + ) + + +async def test_fetch_success_returns_html() -> None: + body = b"Xhi" + resp = _make_stream_response(200, body=body) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + + assert result.ok is True + assert result.status_code == 200 + assert "X" in result.html + assert result.error is None + + +async def test_fetch_timeout() -> None: + with _patch_client(send_side_effect=httpx.TimeoutException("too slow")): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "timeout" + assert result.html == "" + + +async def test_fetch_connect_error() -> None: + with _patch_client(send_side_effect=httpx.ConnectError("dns")): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "connect_error" + + +async def test_fetch_http_4xx() -> None: + resp = _make_stream_response(404, body=b"not found") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error is not None + assert result.error.startswith("http_error") + assert result.status_code == 404 + + +async def test_fetch_http_5xx() -> None: + resp = _make_stream_response(502, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error is not None + assert result.error.startswith("http_error") + + +async def test_fetch_rejects_redirect_status() -> None: + """follow_redirects=False 이므로 3xx 을 받으면 에러로 처리.""" + resp = _make_stream_response(302, headers={"location": "https://other.test/"}) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "unexpected_redirect" + + +async def test_fetch_rejects_content_length_over_cap(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "content_fetch_max_bytes", 1024) + resp = _make_stream_response( + 200, + headers={"content-type": "text/html", "content-length": "999999"}, + ) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "too_large" + + +async def test_fetch_rejects_when_stream_exceeds_cap(monkeypatch: pytest.MonkeyPatch) -> None: + """Content-Length 헤더가 없어도 스트림 누적량이 cap 을 넘으면 too_large 로 거절한다.""" + monkeypatch.setattr(settings, "content_fetch_max_bytes", 64) + + resp = AsyncMock() + resp.status_code = 200 + resp.headers = {"content-type": "text/html"} + resp.charset_encoding = "utf-8" + + async def _aiter_bytes() -> object: + yield b"a" * 100 + yield b"b" * 100 # cap 초과 — 두 번째 청크는 무시되어야 함 + + resp.aiter_bytes = _aiter_bytes + resp.aclose = AsyncMock() + + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + + assert result.ok is False + assert result.error == "too_large" + + +async def test_fetch_rejects_non_html_content_type() -> None: + resp = _make_stream_response(200, headers={"content-type": "image/png"}, body=b"\x89PNG") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is False + assert result.error == "not_html" + + +async def test_fetch_accepts_xhtml() -> None: + resp = _make_stream_response( + 200, + headers={"content-type": "application/xhtml+xml"}, + body=b"x", + ) + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is True + + +async def test_fetch_sets_user_agent_from_settings(monkeypatch: pytest.MonkeyPatch) -> None: + """요청 헤더에 content_user_agents 풀에서 뽑은 UA 가 들어가야 한다.""" + monkeypatch.setattr(settings, "content_user_agents", ["TestUA/1.0"]) + sent_requests: list[httpx.Request] = [] + + async def _capture_send(req: httpx.Request, **_kw: object) -> object: + sent_requests.append(req) + return _make_stream_response(200, body=b"") + + mock_client = AsyncMock() + mock_client.build_request = lambda method, url, **kw: httpx.Request(method, url, **kw) + mock_client.send = _capture_send + + with patch( + "app.services.content_analyzer.fetch._get_client", return_value=mock_client + ): + await fetch_page("https://example.test/") + + assert len(sent_requests) == 1 + assert sent_requests[0].headers["user-agent"] == "TestUA/1.0" + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost/", + "http://127.0.0.1/", + "http://10.0.0.1/", + "http://192.168.1.1/", + "http://169.254.169.254/latest/meta-data/", # AWS IMDS + "http://[::1]/", + "http://metadata.google.internal/", + ], +) +async def test_fetch_rejects_internal_targets(url: str) -> None: + """SSRF 1선 — 사설/loopback/link-local IP 와 잘 알려진 내부 호스트네임은 connect 전에 차단.""" + with patch( + "app.services.content_analyzer.fetch._get_client" + ) as get_client: + result = await fetch_page(url) + assert result.ok is False + assert result.error == "blocked_host" + # 호스트가 차단되면 클라이언트 자체를 가져오지 않는다. + get_client.assert_not_called() + + +async def test_fetch_allows_public_host() -> None: + """공인 IP/도메인은 통과 — 정상 경로 회귀 방지.""" + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is True + + +async def test_fetch_no_follow_redirects_config() -> None: + """모듈 싱글턴 빌더가 follow_redirects=False 로 AsyncClient 를 생성하는지 확인.""" + with patch( + "app.services.content_analyzer.fetch.httpx.AsyncClient" + ) as client_class: + fetch_module._build_client() + kwargs = client_class.call_args.kwargs + assert kwargs.get("follow_redirects") is False + assert kwargs.get("verify") is True + + +class TestSingletonClient: + async def test_get_client_reuses_instance(self) -> None: + """동일 프로세스 안에서 _get_client 는 같은 인스턴스를 돌려줘야 한다 — 풀 재사용 핵심.""" + c1 = fetch_module._get_client() + c2 = fetch_module._get_client() + assert c1 is c2 + await fetch_module.aclose_client() + + async def test_aclose_client_resets_singleton(self) -> None: + """aclose 후 _get_client 는 새 인스턴스를 만든다 — lifespan 재시작 시나리오 보호.""" + c1 = fetch_module._get_client() + await fetch_module.aclose_client() + assert fetch_module._client is None + c2 = fetch_module._get_client() + assert c2 is not c1 + await fetch_module.aclose_client() + + async def test_aclose_client_idempotent(self) -> None: + await fetch_module.aclose_client() + await fetch_module.aclose_client() + + +class TestDnsRebindGuard: + """SSRF 2선 — 호스트네임이 사설 IP 로 풀리면 connect 전에 차단.""" + + async def test_blocks_host_resolving_to_private_ip( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + async def _resolves_to_private(_host: str) -> bool: + return True + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _resolves_to_private, + ) + # 클라이언트 측 send 가 호출되면 검증 실패 — 사전 차단되어야 한다 + with patch( + "app.services.content_analyzer.fetch._get_client" + ) as get_client: + result = await fetch_page("https://evil-public.test/") + assert result.ok is False + assert result.error == "blocked_host" + get_client.assert_not_called() + + async def test_passes_when_resolved_ips_public( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + async def _resolves_public(_host: str) -> bool: + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _resolves_public, + ) + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + result = await fetch_page("https://example.test/") + assert result.ok is True + + async def test_dns_resolution_failure_does_not_block( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """getaddrinfo OSError 는 block 으로 격상하지 않는다 — connect 단계에서 자연 처리.""" + + async def _raises(_host: str) -> bool: + raise OSError("dns down") + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _raises, + ) + with _patch_client(send_side_effect=httpx.ConnectError("dns")): + result = await fetch_page("https://example.test/") + assert result.error == "connect_error" + + async def test_skips_resolution_for_ip_literal( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """공인 IP 리터럴은 1선에서 통과 — 같은 IP 를 다시 해석할 필요 없으므로 2선을 건너뛴다.""" + called: list[str] = [] + + async def _track(host: str) -> bool: + called.append(host) + return False + + monkeypatch.setattr( + "app.services.content_analyzer.fetch._resolved_addrs_blocked", + _track, + ) + resp = _make_stream_response(200, body=b"") + with _patch_client(send_return=resp): + await fetch_page("https://8.8.8.8/") + assert called == [] + + +def test_pick_user_agent_raises_on_empty_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """UA 풀이 비면 침묵 폴백 대신 명시적으로 raise — 운영자가 misconfiguration 을 발견하게.""" + monkeypatch.setattr(settings, "content_user_agents", []) + with pytest.raises(RuntimeError, match="content_user_agents"): + fetch_module._pick_user_agent() diff --git a/tests/services/content_analyzer/test_signals.py b/tests/services/content_analyzer/test_signals.py new file mode 100644 index 0000000..8ae6249 --- /dev/null +++ b/tests/services/content_analyzer/test_signals.py @@ -0,0 +1,182 @@ +"""signals.py — 규칙 기반 점수표.""" + +from __future__ import annotations + +from app.core.config import settings +from app.schemas.content_analysis import ContentSignal +from app.services.content_analyzer.extract import ExtractedFeatures +from app.services.content_analyzer.signals import score_content + + +def _features(**kwargs: object) -> ExtractedFeatures: + base = { + "title": None, + "has_password_field": False, + "has_meta_refresh": False, + "external_link_ratio": None, + "image_alts": [], + } + base.update(kwargs) + return ExtractedFeatures(**base) # type: ignore[arg-type] + + +class TestBrandImpersonationForm: + def test_password_with_brand_in_title_on_unrelated_domain(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + ) + result = score_content(features, final_url="https://evil-naver.test/signin") + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals + assert result.brand_impersonation is True + assert result.score >= settings.score_weight_brand_impersonation + + def test_password_with_brand_matching_own_domain_is_not_impersonation(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + ) + result = score_content(features, final_url="https://naver.com/login") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + assert result.brand_impersonation is False + + def test_password_without_brand_in_title_not_impersonation(self) -> None: + features = _features( + title="일반 로그인 페이지", + has_password_field=True, + ) + result = score_content(features, final_url="https://something.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + def test_no_password_even_with_brand_title_not_brand_form(self) -> None: + features = _features(title="NAVER 소개", has_password_field=False) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + +class TestBrandMatchingFalsePositives: + """영문 라벨은 단어 경계 매칭이라 substring 우연 매치를 일으키지 않아야 한다.""" + + def test_pineapple_does_not_match_apple(self) -> None: + """과거 substring 매칭에서 pineapple → apple 로 잡히던 케이스.""" + features = _features(title="Pineapple Express", has_password_field=True) + result = score_content(features, final_url="https://pineapple-shop.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + assert result.brand_impersonation is False + + def test_naverstore_does_not_match_naver(self) -> None: + features = _features(title="naverstore login", has_password_field=True) + result = score_content(features, final_url="https://other.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM not in result.signals + + def test_googleblog_alt_does_not_match_google(self) -> None: + features = _features(image_alts=["googleblog tutorials"]) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + def test_word_boundary_still_matches_with_punctuation(self) -> None: + # 경계 매칭이라 'NAVER 로그인', 'naver,', 'naver/login' 같은 자연 등장은 그대로 매치 + for title in ["NAVER 로그인", "naver, hello", "Login to naver."]: + features = _features(title=title, has_password_field=True) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.BRAND_IMPERSONATION_FORM in result.signals, title + + +class TestLogoAltImpersonation: + def test_brand_in_alt_on_unrelated_domain(self) -> None: + features = _features(image_alts=["KakaoTalk 로고"]) + result = score_content(features, final_url="https://phish.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.logo_alt_impersonation is True + assert result.score >= settings.score_weight_logo_alt_impersonation + + def test_brand_in_alt_matching_own_domain(self) -> None: + features = _features(image_alts=["kakao logo"]) + result = score_content(features, final_url="https://kakao.com/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + def test_no_alt_text(self) -> None: + features = _features(image_alts=[]) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.LOGO_ALT_IMPERSONATION not in result.signals + + +class TestMetaRefresh: + def test_meta_refresh_adds_score(self) -> None: + features = _features(has_meta_refresh=True) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.META_REFRESH in result.signals + assert result.score == settings.score_weight_meta_refresh + + +class TestExternalLinkOveruse: + def test_above_threshold_signals(self) -> None: + features = _features(external_link_ratio=0.9) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE in result.signals + assert result.score == settings.score_weight_external_link_overuse + + def test_at_threshold_signals(self) -> None: + features = _features( + external_link_ratio=settings.content_external_link_ratio_threshold, + ) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE in result.signals + + def test_below_threshold_no_signal(self) -> None: + features = _features(external_link_ratio=0.3) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE not in result.signals + + def test_none_ratio_no_signal(self) -> None: + features = _features(external_link_ratio=None) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.EXTERNAL_LINK_OVERUSE not in result.signals + + +class TestSpaShellSignal: + def test_spa_shell_emits_signal_without_score(self) -> None: + """판정 불가 플래그라 점수는 0, 시그널 코드만 올라간다.""" + features = _features(is_spa_shell=True) + result = score_content(features, final_url="https://x.test/") + assert ContentSignal.SPA_SHELL in result.signals + assert result.score == 0 + + def test_spa_shell_does_not_interfere_with_other_rules(self) -> None: + """SPA 셸이어도 alt 브랜드 위장 같은 규칙은 정상 집계된다 (드문 케이스지만 방어).""" + features = _features(is_spa_shell=True, image_alts=["NAVER"]) + result = score_content(features, final_url="https://evil.test/") + assert ContentSignal.SPA_SHELL in result.signals + assert ContentSignal.LOGO_ALT_IMPERSONATION in result.signals + assert result.score == settings.score_weight_logo_alt_impersonation + + +class TestCombinedScoring: + def test_all_signals_sum(self) -> None: + features = _features( + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=True, + external_link_ratio=0.95, + image_alts=["NAVER"], + ) + result = score_content(features, final_url="https://evil.test/") + expected = ( + settings.score_weight_brand_impersonation + + settings.score_weight_logo_alt_impersonation + + settings.score_weight_meta_refresh + + settings.score_weight_external_link_overuse + ) + assert result.score == min(expected, settings.content_analysis_score_cap) + + def test_score_cap(self) -> None: + """합산 점수가 cap을 넘지 않는다.""" + features = _features( + title="NAVER 로그인", + has_password_field=True, + has_meta_refresh=True, + external_link_ratio=0.95, + image_alts=["NAVER"], + ) + result = score_content(features, final_url="https://evil.test/") + assert result.score <= settings.content_analysis_score_cap diff --git a/tests/services/test_pipeline.py b/tests/services/test_pipeline.py index 4f79265..82e409b 100644 --- a/tests/services/test_pipeline.py +++ b/tests/services/test_pipeline.py @@ -2,13 +2,16 @@ from __future__ import annotations +import asyncio from unittest.mock import AsyncMock, patch import pytest +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult, ContentSignal from app.schemas.domain_heuristic import DomainHeuristicResult, DomainHeuristicSignal from app.schemas.normalize import NormalizeResult -from app.schemas.pipeline import PipelineFailure, PipelineStage, PipelineSuccess -from app.schemas.threat_db import GSBResult, ThreatDbResult, URLhausResult +from app.schemas.pipeline import PipelineFailure, PipelineStage, PipelineSuccess, Verdict +from app.schemas.threat_db import GSBMatch, GSBResult, ThreatDbResult, URLhausResult from app.schemas.unchain import UnchainResult from app.services.pipeline import run_pipeline from sqlalchemy.ext.asyncio import AsyncSession @@ -38,6 +41,39 @@ def _make_heuristic(domain: str) -> DomainHeuristicResult: ) +def _make_content(final_url: str, *, score: int = 0) -> ContentAnalysisResult: + return ContentAnalysisResult(final_url=final_url, fetched=True, score=score, signals=[]) + + +async def _run_with_scores( + async_session: AsyncSession, + *, + final_url: str, + threat: ThreatDbResult, + heuristic_score: int, + content_score: int = 0, +) -> PipelineSuccess: + """verdict 매핑만 보고 싶을 때 쓰는 헬퍼 — stage 결과는 점수 외엔 의미 없음.""" + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = threat + mock_heuristic.return_value = _heuristic_with_score(heuristic_score) + mock_content.return_value = _make_content(final_url, score=content_score) + + result = await run_pipeline("aid-verdict", final_url, async_session) + assert isinstance(result, PipelineSuccess) + return result + + @pytest.mark.asyncio async def test_run_pipeline_includes_domain_heuristic_stage(async_session: AsyncSession) -> None: """domain_heuristic stage 추가가 응답 스키마에 정상 반영되는지 검증.""" @@ -50,11 +86,13 @@ async def test_run_pipeline_includes_domain_heuristic_stage(async_session: Async patch( "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, ): mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) mock_unchain.return_value = _make_unchain(final_url) mock_threat.return_value = _make_threat(final_url) mock_heuristic.return_value = _make_heuristic("example.com") + mock_content.return_value = _make_content(final_url) result = await run_pipeline("aid-1", final_url, async_session) @@ -63,8 +101,15 @@ async def test_run_pipeline_includes_domain_heuristic_stage(async_session: Async assert result.final_url == final_url assert result.stages.domain_heuristic.domain == "example.com" assert result.stages.domain_heuristic.signals == [DomainHeuristicSignal.HOSTING_PLATFORM] - # threat_db → domain_heuristic 순서 — heuristic 인자는 unchain.final_url + assert result.stages.content_analysis.final_url == final_url + assert result.stages.content_analysis.fetched is True + # threat_db → domain_heuristic → content_analysis 순서 — 모두 unchain.final_url 기준 mock_heuristic.assert_awaited_once_with(final_url) + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + # _make_heuristic 가 HOSTING_PLATFORM 시그널을 가지므로 그대로 전달돼야 한다 + assert kwargs["upstream_signals"] == ("HOSTING_PLATFORM",) @pytest.mark.asyncio @@ -77,6 +122,7 @@ async def test_run_pipeline_normalize_failure_skips_heuristic( with ( patch("app.services.pipeline.normalize_url") as mock_norm, patch("app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock) as mock_h, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_c, ): mock_norm.side_effect = NormalizationError("invalid") @@ -85,3 +131,418 @@ async def test_run_pipeline_normalize_failure_skips_heuristic( assert isinstance(result, PipelineFailure) assert result.failed_at_stage == PipelineStage.NORMALIZE mock_h.assert_not_awaited() + mock_c.assert_not_awaited() + + +def _malicious_threat(final_url: str) -> ThreatDbResult: + return ThreatDbResult( + final_url=final_url, + is_malicious=True, + sources_checked=2, + gsb=GSBResult(checked=True, is_threat=True, matches=[GSBMatch(threat_type="MALWARE")]), + urlhaus=URLhausResult(checked=True, is_threat=False), + threat_types=["MALWARE"], + ) + + +def _heuristic_with_score(score: int) -> DomainHeuristicResult: + return DomainHeuristicResult( + domain="example.com", + score=score, + signals=[], + rdap=None, + rdap_error=None, + ) + + +@pytest.mark.asyncio +async def test_run_pipeline_skips_content_when_gsb_malicious( + async_session: AsyncSession, +) -> None: + """GSB 매치로 is_malicious=True 면 heuristic 점수와 무관하게 4단계를 skip 한다.""" + final_url = "https://evil.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + # GSB(+50) + heuristic(20) = 70 >= 61 → 건너뜀 + mock_heuristic.return_value = _heuristic_with_score(20) + + result = await run_pipeline("aid-skip", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # 네트워크·AI 비용을 아끼기 위해 analyze_content 는 호출되지 않아야 한다 + mock_content.assert_not_awaited() + content = result.stages.content_analysis + assert content.fetched is False + assert content.error == "skipped_already_danger" + assert ContentSignal.SKIPPED_ALREADY_DANGER in content.signals + assert content.score == 0 + + +@pytest.mark.asyncio +async def test_run_pipeline_runs_content_when_below_danger( + async_session: AsyncSession, +) -> None: + final_url = "https://ok.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + # heuristic 점수만으로 임계 미달 → 콘텐츠 분석 정상 수행 + mock_heuristic.return_value = _heuristic_with_score(10) + mock_content.return_value = _make_content(final_url) + + await run_pipeline("aid-run", final_url, async_session) + + # heuristic 시그널 없는 경우 upstream_signals 는 빈 튜플로 전달돼야 한다 + mock_content.assert_awaited_once_with(final_url, upstream_signals=()) + + +class TestVerdictAndScore: + """PipelineSuccess.verdict / score 매핑 회귀.""" + + async def test_safe_when_all_clean(self, async_session: AsyncSession) -> None: + final_url = "https://clean.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=0, + content_score=0, + ) + assert result.verdict == Verdict.SAFE + assert result.score == 0 + + async def test_caution_in_middle_band(self, async_session: AsyncSession) -> None: + """heuristic 35 단독 — caution(31~60) 구간.""" + final_url = "https://mid.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=35, + content_score=0, + ) + assert result.verdict == Verdict.CAUTION + assert result.score == 35 + + async def test_danger_by_score_alone(self, async_session: AsyncSession) -> None: + """blacklist 미스라도 합산 점수가 임계(61) 이상이면 danger.""" + final_url = "https://heur-heavy.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=65, + content_score=0, + ) + assert result.verdict == Verdict.DANGER + assert result.score == 65 + + async def test_score_capped_at_100(self, async_session: AsyncSession) -> None: + """단계별 점수 합이 100 을 넘으면 cap. preceding 임계 미만 → 4단계 합산까지 진입.""" + final_url = "https://overflow.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=10, + content_score=95, + ) + assert result.score == 100 + assert result.verdict == Verdict.DANGER + + async def test_danger_when_threat_matches_even_below_threshold( + self, async_session: AsyncSession + ) -> None: + """GSB 매치 시 합산 점수가 임계 미만이어도 verdict 는 강제로 danger.""" + final_url = "https://blacklist.test/" + # short-circuit 경로 — heuristic 은 placeholder, content skip + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock), + ): + mock_norm.return_value = NormalizeResult( + original_url=final_url, normalized_url=final_url + ) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-blk", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # GSB +50 만 합산되고 임계 61 미달이지만 is_malicious=True 라 verdict=danger + assert result.score == 50 + assert result.verdict == Verdict.DANGER + + async def test_verdict_score_appear_before_stages_in_response( + self, async_session: AsyncSession + ) -> None: + """직렬화 시 verdict/score 가 stages 보다 앞에 나와 클라이언트가 상단에서 바로 읽도록.""" + final_url = "https://order.test/" + result = await _run_with_scores( + async_session, + final_url=final_url, + threat=_make_threat(final_url), + heuristic_score=10, + ) + keys = list(result.model_dump().keys()) + assert keys.index("verdict") < keys.index("stages") + assert keys.index("score") < keys.index("stages") + + +@pytest.mark.asyncio +async def test_run_pipeline_passes_upstream_signals_to_content_analysis( + async_session: AsyncSession, +) -> None: + """heuristic 시그널 + threat 매치 플래그가 analyze_content 의 upstream_signals 로 전달된다.""" + final_url = "https://typo-naverr.test/" + + weak_threat = ThreatDbResult( + final_url=final_url, + is_malicious=False, + sources_checked=2, + # 매치 자체는 False 라 short-circuit 안 일어나지만, 미래에 약한 매치만 있을 케이스 대비 + gsb=GSBResult(checked=True, is_threat=False), + urlhaus=URLhausResult(checked=True, is_threat=False), + ) + heuristic = DomainHeuristicResult( + domain="typo-naverr.test", + score=20, + signals=[DomainHeuristicSignal.TYPO_DOMAIN, DomainHeuristicSignal.NEW_DOMAIN], + rdap=None, + rdap_error=None, + ) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = weak_threat + mock_heuristic.return_value = heuristic + mock_content.return_value = _make_content(final_url) + + await run_pipeline("aid-up", final_url, async_session) + + mock_content.assert_awaited_once() + args, kwargs = mock_content.await_args + assert args == (final_url,) + assert kwargs["upstream_signals"] == ("TYPO_DOMAIN", "NEW_DOMAIN") + + +@pytest.mark.asyncio +async def test_run_pipeline_runs_threat_and_heuristic_in_parallel( + async_session: AsyncSession, +) -> None: + """2·3단계는 서로 독립이므로 gather 로 동시에 진입해야 한다. + + 각 stub 을 상대방의 Event 를 기다리게 만들어서, 순차 실행이면 데드락 → + wait_for 타임아웃으로 실패하도록 유도한다. 병렬이면 둘 다 진입 후 통과. + """ + final_url = "https://ok.test/" + + threat_started = asyncio.Event() + heuristic_started = asyncio.Event() + + async def fake_threat_db(_session: AsyncSession, _url: str) -> ThreatDbResult: + threat_started.set() + await asyncio.wait_for(heuristic_started.wait(), timeout=1.0) + return _make_threat(final_url) + + async def fake_heuristic(_url: str) -> DomainHeuristicResult: + heuristic_started.set() + await asyncio.wait_for(threat_started.wait(), timeout=1.0) + return _heuristic_with_score(10) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=fake_threat_db), + patch("app.services.pipeline.check_domain_heuristic", side_effect=fake_heuristic), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_content.return_value = _make_content(final_url) + + result = await run_pipeline("aid-parallel", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # 둘 다 실행됐고 4단계까지 이어졌는지 확인 + assert threat_started.is_set() and heuristic_started.is_set() + mock_content.assert_awaited_once() + args, _ = mock_content.await_args + assert args == (final_url,) + + +@pytest.mark.asyncio +async def test_run_pipeline_cancelled_error_propagates_from_parallel_stage( + async_session: AsyncSession, +) -> None: + """병렬 단계 중 하나에서 CancelledError 가 나면 degraded 흡수 없이 상위로 올라가야 한다.""" + final_url = "https://ok.test/" + + async def cancelled_threat(_session: AsyncSession, _url: str) -> ThreatDbResult: + raise asyncio.CancelledError + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", side_effect=cancelled_threat), + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_heuristic.return_value = _heuristic_with_score(10) + + with pytest.raises(asyncio.CancelledError): + await run_pipeline("aid-cancel", final_url, async_session) + + +@pytest.mark.asyncio +async def test_run_pipeline_short_circuits_on_gsb_match_and_cancels_heuristic( + async_session: AsyncSession, +) -> None: + """GSB 매치 시: threat_db 결과가 나오자마자 heuristic 을 cancel 하고 + 4단계도 skip 한 채 즉시 종료해야 한다. verdict 가 이미 danger 로 확정이므로. + + placeholder heuristic 의 domain 은 등록 가능 도메인(파이프라인 정상 경로와 동일)이어야 + 하지 full host 가 들어가면 안 된다 — 다운스트림 버킷팅이 어긋난다. + """ + final_url = "https://signin.evil.example.com/login" + + heuristic_finished = asyncio.Event() + + async def slow_heuristic(_url: str) -> DomainHeuristicResult: + # RDAP 대기로 수 초 떠 있는 상황을 흉내낸다. cancel 되지 않으면 이벤트가 set 되어 + # 테스트가 그 사실을 검출한다. + try: + await asyncio.sleep(5.0) + except asyncio.CancelledError: + raise + heuristic_finished.set() + return _heuristic_with_score(10) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch("app.services.pipeline.check_domain_heuristic", side_effect=slow_heuristic), + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _malicious_threat(final_url) + + result = await run_pipeline("aid-sc", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + # heuristic 은 cancel 되어 본문이 끝까지 돌지 않았어야 한다 + assert heuristic_finished.is_set() is False + # 4단계 분석은 skip + mock_content.assert_not_awaited() + content = result.stages.content_analysis + assert content.fetched is False + assert content.error == "skipped_already_danger" + assert ContentSignal.SKIPPED_ALREADY_DANGER in content.signals + # heuristic placeholder 가 채워져 스키마 유지 — domain 은 registrable domain + heur = result.stages.domain_heuristic + assert heur.score == 0 + assert heur.rdap_error == "skipped_threat_matched" + assert heur.domain == "example.com" + + +@pytest.mark.asyncio +async def test_run_pipeline_short_circuits_on_urlhaus_match( + async_session: AsyncSession, +) -> None: + """URLhaus 단독 매치도 동일하게 조기 종료 — threat_db.is_malicious 가 True 이기 때문.""" + final_url = "https://urlhaus-hit.test/" + + urlhaus_only = ThreatDbResult( + final_url=final_url, + is_malicious=True, + sources_checked=2, + gsb=GSBResult(checked=True, is_threat=False), + urlhaus=URLhausResult(checked=True, is_threat=True), + ) + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = urlhaus_only + mock_heuristic.return_value = _heuristic_with_score(0) + + result = await run_pipeline("aid-urlhaus", final_url, async_session) + + assert isinstance(result, PipelineSuccess) + mock_content.assert_not_awaited() + assert result.stages.content_analysis.error == "skipped_already_danger" + + +@pytest.mark.asyncio +async def test_run_pipeline_skips_content_when_heuristic_alone_exceeds_threshold( + async_session: AsyncSession, +) -> None: + """위협 DB 미매치여도 휴리스틱만으로 danger 구간이면 건너뛴다.""" + final_url = "https://typo-naverr.test/" + + with ( + patch("app.services.pipeline.normalize_url") as mock_norm, + patch("app.services.pipeline.unchain_url", new_callable=AsyncMock) as mock_unchain, + patch("app.services.pipeline.check_threat_db", new_callable=AsyncMock) as mock_threat, + patch( + "app.services.pipeline.check_domain_heuristic", new_callable=AsyncMock + ) as mock_heuristic, + patch("app.services.pipeline.analyze_content", new_callable=AsyncMock) as mock_content, + ): + mock_norm.return_value = NormalizeResult(original_url=final_url, normalized_url=final_url) + mock_unchain.return_value = _make_unchain(final_url) + mock_threat.return_value = _make_threat(final_url) + mock_heuristic.return_value = _heuristic_with_score(settings.score_danger_threshold) + + await run_pipeline("aid-heur", final_url, async_session) + + mock_content.assert_not_awaited() From 4e93bfc619ab92e00e589ef9a3e13ca7bb9d821b Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:41:23 +0900 Subject: [PATCH 088/173] =?UTF-8?q?=20[Chore]=20=EC=BD=98=ED=85=90?= =?UTF-8?q?=EC=B8=A0=20=EC=A0=95=EC=A0=81=20=EB=B6=84=EC=84=9D=20DEMO=20?= =?UTF-8?q?=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/demo/demo_content_analysis.py | 107 ++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/demo/demo_content_analysis.py diff --git a/tests/demo/demo_content_analysis.py b/tests/demo/demo_content_analysis.py new file mode 100644 index 0000000..2bf0669 --- /dev/null +++ b/tests/demo/demo_content_analysis.py @@ -0,0 +1,107 @@ +"""페이지 콘텐츠 정적 분석 데모. + +사용법: + python -m tests.demo.demo_content_analysis + +실제 네트워크로 HTML 을 취득해 BS4 로 파싱하고 규칙 기반 점수를 계산한다. +AI 어댑터는 기본값 `NullAIProvider` — `OPENAI_API_KEY` 환경변수가 있으면 +이 스크립트가 `OpenAIProvider` 로 교체해서 gpt-4o-mini 추론도 같이 보여준다. + +외부 사이트가 응답하지 않으면 해당 케이스는 `fetched=False`, `error=...` 로 +degraded 반환되고 파이프라인은 계속 돈다 (의도된 동작). +""" + +from __future__ import annotations + +import asyncio + +from app.core.config import settings +from app.schemas.content_analysis import ContentAnalysisResult +from app.services.content_analyzer import analyze_content +from app.services.content_analyzer.ai import set_ai_provider +from app.services.content_analyzer.ai_openai import OpenAIProvider + +DIVIDER = "-" * 72 + + +CASES: list[tuple[str, str]] = [ + # --- 정상 페이지 --- + ("정상 페이지 (example.com)", "https://example.com/"), + ("자기 브랜드 도메인 (naver.com)", "https://www.naver.com/"), + # --- 페치/파싱 경로 점검 --- + ("HTML 샘플 (httpbin)", "https://httpbin.org/html"), + ("비-HTML content-type (favicon)", "https://www.google.com/favicon.ico"), + # --- degraded 경로 --- + ( + "존재하지 않는 호스트 (connect_error 기대)", + "https://this-host-definitely-does-not-exist-xyzabc-123.test/", + ), + # --- 리다이렉트 방지 동작 확인 --- + ( + "3xx 응답 (unexpected_redirect — unchainer 가 먼저 해소했어야 하는 케이스)", + "http://google.com/", + ), +] + + +def _fmt_ratio(r: float | None) -> str: + return "-" if r is None else f"{r:.2f}" + + +def _print_result(label: str, url: str, result: ContentAnalysisResult) -> None: + print(f" input : {url}") + print(f" fetched : {result.fetched}") + if not result.fetched: + print(f" error : {result.error}") + else: + print(f" title : {result.title!r}") + print(f" password_form : {result.has_password_field}") + print(f" meta_refresh : {result.has_meta_refresh}") + print(f" ext_link_ratio: {_fmt_ratio(result.external_link_ratio)}") + print(f" brand_imperson: {result.brand_impersonation}") + print(f" alt_imperson : {result.logo_alt_impersonation}") + print(f" score : {result.score}") + print(f" signals : {[s.value for s in result.signals]}") + verdict = result.ai_verdict.value if result.ai_verdict else None + print(f" ai_verdict : {verdict} (reason={result.ai_reason!r})") + if result.ai_error: + print(f" ai_error : {result.ai_error}") + + +async def main() -> None: + print() + print("=" * 72) + print(" LinClean -- Content Analysis Demo") + print("=" * 72) + print() + + provider: OpenAIProvider | None = None + if settings.openai_api_key: + provider = OpenAIProvider() + set_ai_provider(provider) + print(f" AI: OpenAIProvider ({settings.openai_model})") + else: + print(" AI: NullAIProvider (OPENAI_API_KEY 미설정 — ai_verdict 는 None)") + print(f" danger threshold (skip 기준): {settings.score_danger_threshold}") + print() + + try: + for i, (desc, url) in enumerate(CASES, 1): + print(f" [{i:02d}] {desc}") + try: + result = await analyze_content(url) + except Exception as exc: # 데모에서만 — 운영 코드는 파이프라인이 degraded 처리 + print(f" !! 예외: {type(exc).__name__}: {exc}") + print(DIVIDER) + continue + _print_result(desc, url, result) + print(DIVIDER) + finally: + if provider is not None: + await provider.aclose() + + print() + + +if __name__ == "__main__": + asyncio.run(main()) From da25738a0c21c214ea1c9246601493edaf52b411 Mon Sep 17 00:00:00 2001 From: minsoo0506 Date: Tue, 28 Apr 2026 10:41:41 +0900 Subject: [PATCH 089/173] =?UTF-8?q?[Docs]=204=EB=8B=A8=EA=B3=84=20?= =?UTF-8?q?=EC=BD=98=ED=85=90=EC=B8=A0=20=EB=B6=84=EC=84=9D=20=C2=B7=20?= =?UTF-8?q?=EB=A9=94=EB=AA=A8=EB=A6=AC=20=EA=B0=80=EC=9D=B4=EB=93=9C=20?= =?UTF-8?q?=C2=B7=20/api/v1/dev/*=20=EB=94=94=EB=B2=84=EA=B7=B8=20?= =?UTF-8?q?=EC=97=94=EB=93=9C=ED=8F=AC=EC=9D=B8=ED=8A=B8=20=EC=84=B9?= =?UTF-8?q?=EC=85=98=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 239 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 231 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 48f8938..5ff0921 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ | Type Check | **mypy (strict)** | pydantic 플러그인 | | Test | **pytest + pytest-asyncio + httpx AsyncClient** | ASGI 테스트 | | Packaging | **hatchling** | PEP 621 | -| AI | **모델 미정** | 페이지 콘텐츠 정적 분석 보조 | +| AI | **OpenAI Chat Completions (gpt-4o-mini 기본)** | 페이지 콘텐츠 정적 분석 보조 — 모델은 `OPENAI_MODEL` 로 교체 | --- @@ -244,6 +244,23 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 규칙 기반 점수표로 도메인의 위험 신호를 합산합니다. 도메인 등록 정보는 **RDAP (RFC 7480~7484)** 로 조회합니다. +**2단계와 동시 실행 + 외부 DB 매치 시 조기 종료**: 두 단계 모두 1단계 최종 URL +만 필요하고 서로 독립이라 `run_pipeline` 에서 두 task 를 동시에 띄우고 +`asyncio.wait(return_when=FIRST_COMPLETED)` 로 먼저 끝난 쪽을 확인합니다. + +- **GSB 또는 URLhaus 매치 (`threat_db.is_malicious=True`)** 가 먼저 떨어지면, + 아직 RDAP 대기 중일 수 있는 heuristic task 를 **즉시 `cancel()`** 하고 4단계도 + `skipped_already_danger` 로 묶어 바로 반환합니다. verdict 가 이미 danger 로 + 확정이므로 RDAP 이 돌아올 때까지 대기할 이유가 없습니다. heuristic 자리에는 + `rdap_error="skipped_threat_matched"` 인 placeholder 가 채워져 응답 스키마를 + 유지합니다. +- **heuristic 이 먼저 끝난 경우**는 threat_db 를 마저 기다린 뒤, is_malicious + 이거나 합산 점수가 임계를 넘으면 4단계만 skip 합니다. +- **정상 경로**에서는 GSB(100~300ms)와 RDAP(캐시 미스 시 최대 ~5s)의 latency 가 + 겹쳐서 4단계 skip 판정까지의 wall clock 이 둘 중 느린 쪽으로 수렴합니다. +- `CancelledError` 와 stage 내부 예외는 남은 task 를 정리한 뒤 상위로 전파되어 + shutdown / 타임아웃 신호가 degraded 결과로 삼켜지지 않습니다. + | 검사 항목 | 위험 신호 예시 | 점수 | |----------|----------------|------| | IP 직접 접근 | `http://192.168.x.x/login` | +40 | @@ -264,9 +281,9 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 ### 4단계 — 페이지 콘텐츠 정적 분석 -`requests + BeautifulSoup` 으로 실제 페이지를 크롤링해 HTML 구조를 추출하고, -**AI(모델 미정) API** 가 피싱 신호를 정적으로 추론합니다. AI 는 본 엔진 안에서 -이 정적 분석 단계에서만 사용됩니다. +`httpx + BeautifulSoup` 으로 실제 페이지를 크롤링해 HTML 구조를 추출하고, +**OpenAI Chat Completions API** 가 피싱 신호를 정적으로 추론합니다. AI 는 본 +엔진 안에서 이 정적 분석 단계에서만 사용됩니다. 추출 / 점수화하는 신호: @@ -277,12 +294,118 @@ upsert 합니다. 분석 시에는 외부 호출 없이 로컬 인덱스만 조 - **외부 링크 비율 과다**: 80% 이상이 외부 도메인이면 (+15) - **AI 정적 추론**: 추출된 텍스트·폼·메타데이터를 AI 에게 넘겨 "이 페이지가 특정 브랜드를 사칭하거나 자격 증명을 탈취하려 하는지" 여부와 근거 텍스트를 - 반환받아 점수에 반영 -- 페이지 접근 실패 시 +20 점 (의심 신호로 취급) + 반환받아 점수에 반영 (phishing +40 / suspicious +20 / benign 0) +- **`SPA_SHELL`** (점수 0, 시그널만): 초기 HTML 이 React/Vue/Next/Nuxt/Svelte/Angular + 마운트 셸뿐이라 정적 추출로 폼·입력 판정이 결정적이지 않은 상태. `is_spa_shell=true` + 로 응답에 실리고, AI 프롬프트에도 힌트로 전달돼 모델이 "폼 없음" 으로 단정하지 + 않도록 한다. 정상 SPA 가 압도적으로 많아 점수 가산은 하지 않는다. +- **페이지 접근 실패** — 사유별 가산 분리: + - `timeout` / `connect_error` / `http_error_*` / `unexpected` (도달 자체 실패): **+10** + - `not_html` (PDF/이미지 등 정상 비-HTML), `too_large` (대용량 정상 페이지), + `unexpected_redirect` (unchainer 가 놓친 3xx — 파이프라인 정합성 이슈): **+0** (시그널만) + - `blocked_host` (사설/loopback IP 또는 클라우드 메타데이터 호스트): **+10** (SSRF 1선 차단) + +#### 브랜드 매칭 전략 — 영문은 단어 경계, 한국어는 substring + +`brands.txt` (약 600개) 의 라벨로 title/alt 텍스트를 매칭할 때, 영문 라벨은 +`\b