From b7eb4cf4a343d2d7c8cf60ab8644cd4a16d74157 Mon Sep 17 00:00:00 2001 From: Simone P Date: Fri, 12 Jun 2026 13:58:52 +0000 Subject: [PATCH 1/4] Improve scraper reliability, config safety, and add GitLab CI - Add .gitlab-ci.yml running black, flake8, mypy and pytest with coverage - Fix price regex to support prices >= 1000 (was capped at 3 digits) - Static scraper: skip failed pages with delay instead of hot-looping the same URL - Browser scraper: remove bare except, log skipped cards, preserve tracebacks - config.py: tolerate malformed numeric env vars with safe fallbacks - main.py: fall back to DB data for terminal dashboard when no new products - README: add Responsible Scraping guidance - Tests for new behavior (price parsing, error path, config helpers) --- .env.example | 3 +++ .gitlab-ci.yml | 41 ++++++++++++++++++++++++++++++ README.md | 12 +++++++++ config.py | 29 ++++++++++++++++++--- main.py | 11 +++++--- scraper/product_scraper.py | 10 +++++--- scraper/product_scraper_browser.py | 13 +++++++--- tests/test_config.py | 30 ++++++++++++++++++++++ tests/test_scraper.py | 24 +++++++++++++++++ 9 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 tests/test_config.py diff --git a/.env.example b/.env.example index 108fd1f..f73adfd 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,7 @@ BASE_URL=https://sandbox.oxylabs.io/products MAX_RETRIES=3 DEFAULT_TIMEOUT=30 +RETRY_BACKOFF=0.5 DB_NAME=products.db +# Optional: override the User-Agent header sent by the scrapers +# USER_AGENT="MyScraperBot/1.0 (+https://example.com/contact)" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..3c05cf5 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,41 @@ +stages: + - lint + - test + +default: + image: python:3.11-slim + before_script: + - python -m pip install --upgrade pip + - pip install -r requirements.txt + - pip install -e .[dev] + cache: + key: + files: + - requirements.txt + paths: + - .cache/pip + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + +lint: + stage: lint + script: + - black --check . + - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + +typecheck: + stage: lint + script: + - mypy cleaning scraper visualization *.py --ignore-missing-imports + +test: + stage: test + script: + - pytest --cov=scraper --cov=cleaning --cov=visualization --cov-report=term-missing --junitxml=report.xml + coverage: '/TOTAL.*\s+(\d+%)$/' + artifacts: + when: always + reports: + junit: report.xml diff --git a/README.md b/README.md index 3b0751c..d241dc6 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,18 @@ The `products_powerbi.csv` file is formatted for seamless Power BI import: --- +## 🤝 Responsible Scraping + +This project targets a public scraping sandbox explicitly intended for practice. When adapting it to other sites: + +- **Check `robots.txt` and the site's Terms of Service** before scraping +- **Keep a delay between requests** (`--delay`, default 1s) and do not raise the browser concurrency limit +- **Identify your client honestly** via the `USER_AGENT` env var where appropriate +- **Never scrape private, personal, or paywalled data**, and never bypass authentication or captchas +- **Fail gracefully**: both scrapers back off and stop after repeated failures instead of hammering the server + +--- + ## ⚠️ Known Limitations - **Static scraper vs. JS-rendered sites**: The `static` scraper uses `requests` + BeautifulSoup, which cannot execute JavaScript. The target sandbox site is JS-rendered, so **use `--type browser`** for actual scraping. The static scraper is included to demonstrate the pattern and works with server-rendered HTML. diff --git a/config.py b/config.py index 7bf6357..7b3f731 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,29 @@ # Load environment variables from .env file load_dotenv() + +def _env_int(name: str, default: int) -> int: + """Read an integer env var, falling back to default on invalid values.""" + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except ValueError: + return default + + +def _env_float(name: str, default: float) -> float: + """Read a float env var, falling back to default on invalid values.""" + raw = os.getenv(name) + if raw is None: + return default + try: + return float(raw) + except ValueError: + return default + + # Base Paths BASE_DIR = Path(__file__).parent DATA_DIR = BASE_DIR / "data" @@ -30,9 +53,9 @@ # Scraper Settings BASE_URL = os.getenv("BASE_URL", "https://sandbox.oxylabs.io/products") -DEFAULT_TIMEOUT = int(os.getenv("DEFAULT_TIMEOUT", 30)) -MAX_RETRIES = int(os.getenv("MAX_RETRIES", 3)) -RETRY_BACKOFF = float(os.getenv("RETRY_BACKOFF", 0.5)) +DEFAULT_TIMEOUT = _env_int("DEFAULT_TIMEOUT", 30) +MAX_RETRIES = _env_int("MAX_RETRIES", 3) +RETRY_BACKOFF = _env_float("RETRY_BACKOFF", 0.5) USER_AGENT_FALLBACK = os.getenv( "USER_AGENT", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", diff --git a/main.py b/main.py index eb7fd48..634732e 100644 --- a/main.py +++ b/main.py @@ -98,10 +98,13 @@ def scrape( if cleaned_products: generate_terminal_dashboard(cleaned_products) else: - # If no new products, try to get from DB for terminal dashboard - # Note: generate_terminal_dashboard expects a list of Product objects. - # We might need to fetch them if cleaned_products is empty but DB has data. - pass + # If no new products, fall back to existing DB data + try: + existing = db.get_all_products() + if existing: + generate_terminal_dashboard(existing) + except Exception as e: + logger.warning(f"Could not generate terminal dashboard from DB: {e}") duration = time.time() - start_time logger.info(f"Pipeline completed in {duration:.2f} seconds.") diff --git a/scraper/product_scraper.py b/scraper/product_scraper.py index 6493a76..52f13c0 100644 --- a/scraper/product_scraper.py +++ b/scraper/product_scraper.py @@ -4,6 +4,7 @@ Uses Requests and BeautifulSoup to scrape static HTML. """ +import re import time from typing import List, Optional import requests @@ -72,7 +73,12 @@ def scrape(self, max_pages: Optional[int] = None) -> List[Product]: logger.error(f"Error scraping {url}: {e}") consecutive_empty += 1 if consecutive_empty >= 3: + logger.info("Stopping: 3 consecutive failed pages.") break + # Skip the failing page instead of hot-looping on it, + # and back off politely before the next request. + page += 1 + time.sleep(self.delay) return all_products @@ -116,10 +122,8 @@ def _parse_page(self, content: bytes, url: str) -> List[Product]: @staticmethod def _extract_price(card: Tag) -> float: """Extract price from a product card using regex.""" - import re - text = card.get_text(separator=" ", strip=True) - match = re.search(r"(\d{1,3}(?:[.,]\d{2})?)\s*€", text) + match = re.search(r"(\d+(?:[.,]\d{2})?)\s*€", text) if match: return float(match.group(1).replace(",", ".")) return 0.0 diff --git a/scraper/product_scraper_browser.py b/scraper/product_scraper_browser.py index 2bdd839..78c910d 100644 --- a/scraper/product_scraper_browser.py +++ b/scraper/product_scraper_browser.py @@ -7,7 +7,11 @@ import asyncio import re from typing import List, Optional -from playwright.async_api import async_playwright, Page, BrowserContext +from playwright.async_api import ( + async_playwright, + BrowserContext, + TimeoutError as PlaywrightTimeoutError, +) from scraper.base import BaseScraper from models import Product @@ -37,7 +41,7 @@ def scrape(self, max_pages: Optional[int] = None) -> List[Product]: @staticmethod def _extract_price(text: str) -> float: """Extract price from card text using regex.""" - match = re.search(r"(\d{1,3}(?:[.,]\d{2})?)\s*€", text) + match = re.search(r"(\d+(?:[.,]\d{2})?)\s*€", text) if match: return float(match.group(1).replace(",", ".")) return 0.0 @@ -146,7 +150,7 @@ async def _scrape_single_page( # Wait for content to render try: await page.wait_for_selector("div.product-card", timeout=5000) - except: + except PlaywrightTimeoutError: # If timeout, we proceed to count (which will be 0) pass @@ -189,11 +193,12 @@ async def _scrape_single_page( ) ) except Exception as e: + logger.debug(f"Skipping unparsable card on {url}: {e}") continue except Exception as e: logger.error(f"Failed to scrape {url}: {e}") - raise e + raise finally: await page.close() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..1551824 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,30 @@ +""" +Tests for configuration helpers. +""" + +from config import _env_int, _env_float + + +def test_env_int_returns_default_when_unset(monkeypatch): + monkeypatch.delenv("SOME_INT", raising=False) + assert _env_int("SOME_INT", 7) == 7 + + +def test_env_int_parses_valid_value(monkeypatch): + monkeypatch.setenv("SOME_INT", "42") + assert _env_int("SOME_INT", 7) == 42 + + +def test_env_int_falls_back_on_invalid_value(monkeypatch): + monkeypatch.setenv("SOME_INT", "not-a-number") + assert _env_int("SOME_INT", 7) == 7 + + +def test_env_float_parses_valid_value(monkeypatch): + monkeypatch.setenv("SOME_FLOAT", "1.5") + assert _env_float("SOME_FLOAT", 0.5) == 1.5 + + +def test_env_float_falls_back_on_invalid_value(monkeypatch): + monkeypatch.setenv("SOME_FLOAT", "oops") + assert _env_float("SOME_FLOAT", 0.5) == 0.5 diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 2f88a7e..7623b27 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -178,3 +178,27 @@ def test_extract_availability_unknown(): html = '

Test

Something else
' card = BeautifulSoup(html, "html.parser").find("div") assert StaticScraper._extract_availability(card) == "Unknown" + + +def test_extract_price_four_digits(): + """Test price extraction supports prices of 1000 or more.""" + from bs4 import BeautifulSoup + + html = '

Test

1059.99 €
' + card = BeautifulSoup(html, "html.parser").find("div") + assert StaticScraper._extract_price(card) == 1059.99 + + +@patch("scraper.product_scraper.StaticScraper._create_session") +def test_static_scraper_error_skips_page(mock_session): + """Test that a failing page is skipped instead of retried in a tight loop.""" + session = Mock() + session.get.side_effect = Exception("Connection error") + mock_session.return_value = session + + scraper = StaticScraper(base_url="http://test.com", delay=0) + products = scraper.scrape(max_pages=1) + + assert products == [] + # Page counter advances on error, so with max_pages=1 only one request is made + assert session.get.call_count == 1 From 92d143d495877f91d50d934fdc1762fe644c0725 Mon Sep 17 00:00:00 2001 From: Simone P Date: Fri, 12 Jun 2026 14:07:39 +0000 Subject: [PATCH 2/4] ci: add workflow rules so jobs run in merge request pipelines --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c05cf5..8f5cc15 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,13 @@ +workflow: + rules: + # Run merge request pipelines + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Avoid duplicate pipelines when a branch has an open MR + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + # Run branch pipelines (e.g. main) + - if: $CI_COMMIT_BRANCH + stages: - lint - test From 9b11ce49aa8289b587daf7ecde031c66ae84fc8e Mon Sep 17 00:00:00 2001 From: Simone P Date: Fri, 12 Jun 2026 14:50:43 +0000 Subject: [PATCH 3/4] Redesign modern dashboard with filters, CSV export and insights - Visual refresh: aurora gradient backdrop, glassmorphism panels, sticky section nav, hero headline, refined KPI cards with animated counters - New features: availability + price-range filters on the product table, client-side CSV export of the filtered view, availability doughnut chart, top-10 most expensive leaderboard with animated bars - Accessibility: aria labels, system-aware dark mode persistence, prefers-reduced-motion support, graceful empty state - Generator passes new stats (min/max price, availability breakdown, top products, ISO timestamp) keeping the public API unchanged - Add tests for dashboard generation --- README.md | 2 +- tests/test_dashboard.py | 31 ++ visualization/dashboard_generator.py | 31 ++ .../templates/dashboard_modern_template.html | 459 +++++++++++------- 4 files changed, 346 insertions(+), 177 deletions(-) create mode 100644 tests/test_dashboard.py diff --git a/README.md b/README.md index d241dc6..f8dea91 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ pytest --cov=scraper --cov=cleaning --cov=visualization --cov-report=term-missin ### Visualization (`visualization/`) -- **Modern Dashboard**: Tailwind CSS, Chart.js (price distribution, segment doughnut), Grid.js (searchable/sortable product table), Alpine.js (dark mode toggle) +- **Modern Dashboard**: Tailwind CSS glass UI with aurora backdrop, Chart.js (price histogram, availability doughnut), Grid.js table with search, sorting, availability/price-range filters and client-side CSV export, top-10 price leaderboard, animated KPI counters, Alpine.js dark/light mode (system-aware, persisted), reduced-motion and aria support - **Terminal Dashboard**: Retro CRT-style with ASCII bar charts, auto-calculated KPIs - Auto-detected franchise/keyword analysis (no hardcoded keywords) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 0000000..276e925 --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,31 @@ +""" +Tests for the dashboard generator. +""" + +from visualization.dashboard_generator import generate_dashboard + + +def test_generate_dashboard_creates_file(tmp_path, db_manager, sample_products): + """Dashboard HTML is rendered with product data embedded.""" + db_manager.save_products(sample_products) + output = tmp_path / "dashboard.html" + + result = generate_dashboard(db_manager, output_path=str(output)) + + assert result == str(output) + assert output.exists() + html = output.read_text(encoding="utf-8") + assert "ScrapingStore" in html + assert "Zelda: Breath of the Wild" in html + # New stats are embedded for the redesigned dashboard + assert "top-products-data" in html + assert "availability-data" in html + + +def test_generate_dashboard_empty_db(tmp_path, db_manager): + """No file is written when the database has no products.""" + output = tmp_path / "dashboard.html" + + generate_dashboard(db_manager, output_path=str(output)) + + assert not output.exists() diff --git a/visualization/dashboard_generator.py b/visualization/dashboard_generator.py index 757868a..04dfd24 100644 --- a/visualization/dashboard_generator.py +++ b/visualization/dashboard_generator.py @@ -65,6 +65,13 @@ def _detect_franchises(df: pd.DataFrame, top_n: int = 8) -> List[Dict[str, Any]] return franchises +def _top_products(df: pd.DataFrame, n: int = 10) -> List[Dict[str, Any]]: + """Return the n most expensive products for the insights panel.""" + cols = [c for c in ("name", "price", "availability") if c in df.columns] + top = df.sort_values("price", ascending=False).head(n) + return [dict(rec) for rec in top[cols].to_dict(orient="records")] + + def generate_dashboard( db: DatabaseManager, output_path: str = str(DASHBOARD_HTML_PATH) ) -> str: @@ -111,6 +118,25 @@ def generate_dashboard( franchise_data: List[Dict[str, Any]] = _detect_franchises(df) franchise_data.sort(key=lambda x: int(x["count"]), reverse=True) + # 2b. Additional stats for the redesigned dashboard + min_price = float(df["price"].min()) + max_price = float(df["price"].max()) + + out_of_stock_count = 0 + if "availability" in df.columns: + out_of_stock_count = int( + df["availability"].str.contains("Out of Stock", case=False, na=False).sum() + ) + unknown_count = max(total_products - in_stock_count - out_of_stock_count, 0) + availability_json = json.dumps( + { + "in_stock": in_stock_count, + "out_of_stock": out_of_stock_count, + "unknown": unknown_count, + } + ) + top_products_json = json.dumps(_top_products(df), default=str) + # 3. Price Distribution (Dynamic Bins) # create ~8 bins based on min-max hist_labels = [] @@ -158,14 +184,19 @@ def generate_dashboard( context = { "timestamp": datetime.now().strftime("%b %d, %Y • %H:%M"), + "generated_iso": datetime.now().astimezone().isoformat(timespec="seconds"), "products_json": products_json, "franchise_json": franchise_json, "kpi_total": total_products, "kpi_avg": f"{avg_price:.2f}", + "kpi_min": f"{min_price:.2f}", + "kpi_max": f"{max_price:.2f}", "kpi_premium": premium_count, "kpi_availability_pct": availability_pct, "kpi_availability_label": availability_label, "chart_data_json": chart_json, + "availability_json": availability_json, + "top_products_json": top_products_json, } html_content = template.render(context) diff --git a/visualization/templates/dashboard_modern_template.html b/visualization/templates/dashboard_modern_template.html index 37a7977..2b0f638 100644 --- a/visualization/templates/dashboard_modern_template.html +++ b/visualization/templates/dashboard_modern_template.html @@ -5,34 +5,28 @@ ScrapingStore Analytics + - + - @@ -43,83 +37,90 @@ + -
-
+

Initializing data stream...

- -
-
-
- S -
-
-

ScrapingStore

-

Analytics Dashboard

+ +
- + + +
+

Product Intelligence

+

Market snapshot, beautifully decoded.

+

{{ kpi_total }} products scraped, cleaned and analyzed by the ScrapingStore pipeline. Last updated .

+
+ +

Total Products

-

0

+

0

Scraped successfully

Avg Price

-

€0

-

Market average

+

€0

+

Range €{{ kpi_min }} – €{{ kpi_max }}

Premium Items

-

0

-

Products > €85

+

0

+

Products > €85

Availability

-

0%

-

—

+

0%

+

{{ kpi_availability_label }}

- -
-
+ +
+
-

Price Distribution

+

Price Distribution

Histogram
-
+
-
+
-

Segments

- Breakdown +

Availability

+ Stock split
-
+
-
+
+ + +
+
+

Product Catalog

+ +
+
+ + + + +
+ +
+ +
+
- -
-
+ +
+
-

Product Catalog

- 0 items +

Top 10 Most Expensive

+ Leaderboard
-
+
    + +
-
+
-

Top Keywords

+

Top Keywords

Auto-detected
-