diff --git a/docs/usage/information_provider_system.md b/docs/usage/information_provider_system.md index fec153a2a..cd40733f5 100644 --- a/docs/usage/information_provider_system.md +++ b/docs/usage/information_provider_system.md @@ -333,6 +333,23 @@ This is not an official API and could break at any time. So use it at your own r The following env configuration options are available: * `PROVIDER_POLLIN_ENABLED`: Set this to `1` to enable the Pollin provider +### Pololu + +The Pololu provider uses webscraping from [pololu.com](https://www.pololu.com/) to get part information. +This is not an official API and could break at any time. So use it at your own risk. + +You can search by keyword or by the Pololu item number (e.g. `2130`). Besides the basic infos, the provider returns +all product pictures, the price breaks (in USD), the available stock, the files from the "Resources" tab (datasheets, +dimension diagrams, 3D models, drill guides, ...) and the specifications from the "Specs" tab as parameters. + +Getting the details of a product needs up to three page requests (product page, specs and resources). To be polite to +the shop, the provider waits a configurable time between two page requests, and stops sending requests for some time +when the website answers with an error indicating that requests are blocked or rate limited (HTTP 403, 429 or 503). + +The following env configuration options are available: +* `PROVIDER_POLOLU_ENABLED`: Set this to `1` to enable the Pololu provider +* `PROVIDER_POLOLU_REQUEST_DELAY`: The minimum time in seconds between two page requests to pololu.com (optional, default: `1`) + ### Buerklin The Buerklin provider uses the [Buerklin API](https://www.buerklin.com/en/services/eprocurement/) to search for parts and get information. diff --git a/src/Services/InfoProviderSystem/Providers/PololuProvider.php b/src/Services/InfoProviderSystem/Providers/PololuProvider.php new file mode 100644 index 000000000..7d15cbd73 --- /dev/null +++ b/src/Services/InfoProviderSystem/Providers/PololuProvider.php @@ -0,0 +1,579 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem\Providers; + +use App\Entity\Parts\ManufacturingStatus; +use App\Services\InfoProviderSystem\DTOs\FileDTO; +use App\Services\InfoProviderSystem\DTOs\ParameterDTO; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\PriceDTO; +use App\Services\InfoProviderSystem\DTOs\ProviderInfoDTO; +use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Settings\InfoProviderSystem\PololuSettings; +use Psr\Cache\CacheItemPoolInterface; +use Symfony\Component\DomCrawler\Crawler; +use Symfony\Contracts\HttpClient\HttpClientInterface; +use Symfony\Contracts\HttpClient\ResponseInterface; + +/** + * Retrieves part information from pololu.com by scraping the product pages. + * + * The provider id is the Pololu item number, which is also part of the product URL (/product/). + * The specifications and the downloadable files are on separate tabs of a product (/product//specs and + * /product//resources), which are only requested if the product page says that they are not empty. + */ +class PololuProvider implements InfoProviderInterface, URLHandlerInfoProviderInterface +{ + public const PROVIDER_KEY = 'pololu'; + + public const DISTRIBUTOR_NAME = 'Pololu'; + + private const BASE_URL = 'https://www.pololu.com'; + + /** @var string All prices on pololu.com are in US dollars */ + private const CURRENCY = 'USD'; + + /** @var int[] The HTTP status codes which indicate that the website does not want our requests (anymore) */ + private const BLOCKED_STATUS_CODES = [403, 429, 503]; + + /** @var int How long (in seconds) no requests are sent anymore, after the website has blocked a request */ + private const BLOCKED_PAUSE = 3600; + + private const CACHE_KEY_BLOCKED = 'pololu_blocked'; + private const CACHE_KEY_LAST_REQUEST = 'pololu_last_request'; + + /** @var string Links outside the file downloads list are only used, if they point to a file with one of these extensions */ + private const FILE_EXTENSIONS_REGEX = '/\.(pdf|zip|step|stp|igs|iges|stl|dxf|dwg|sldprt|easm|eprt|brd|sch|hex|bin)$/i'; + + public function __construct(private readonly HttpClientInterface $client, + private readonly PololuSettings $settings, + private readonly CacheItemPoolInterface $partInfoCache, + ) + { + } + + public function getProviderInfo(): ProviderInfoDTO + { + return new ProviderInfoDTO( + key: self::PROVIDER_KEY, + name: 'Pololu', + description: 'Webscraping from pololu.com to get part information', + url: 'https://www.pololu.com/', + disabledHelp: 'Enable the provider in provider settings', + settingsClass: PololuSettings::class, + capabilities: [ + ProviderCapabilities::BASIC, + ProviderCapabilities::PICTURE, + ProviderCapabilities::PRICE, + ProviderCapabilities::DATASHEET, + ProviderCapabilities::PARAMETERS, + ProviderCapabilities::STOCK_LEVEL, + ], + ); + } + + public function isActive(): bool + { + return $this->settings->enabled; + } + + /** + * Requests the given page of pololu.com. The requests are paced, and if the website blocks a request, no + * further requests are sent for some time. + * @param string $path The path of the page, starting with a slash + * @param array $query + */ + private function request(string $path, array $query = []): ResponseInterface + { + if ($this->partInfoCache->getItem(self::CACHE_KEY_BLOCKED)->isHit()) { + throw new \RuntimeException('pololu.com has blocked a previous request. No requests are sent to it for some time, try again later.'); + } + + $this->waitForNextRequest(); + + $response = $this->client->request('GET', self::BASE_URL . $path, [ + 'query' => $query, + ]); + + if (in_array($response->getStatusCode(), self::BLOCKED_STATUS_CODES, true)) { + $item = $this->partInfoCache->getItem(self::CACHE_KEY_BLOCKED); + $item->set(time()); + $item->expiresAfter(self::BLOCKED_PAUSE); + $this->partInfoCache->save($item); + + throw new \RuntimeException(sprintf('pololu.com has blocked the request (HTTP status %d). No requests are sent to it for some time, try again later.', $response->getStatusCode())); + } + + return $response; + } + + /** + * Sleeps until the configured delay since the last request has passed. The time of the last request is stored + * in the cache, so the delay is also respected between different lookups. + */ + private function waitForNextRequest(): void + { + $delay = $this->settings->requestDelay; + $item = $this->partInfoCache->getItem(self::CACHE_KEY_LAST_REQUEST); + + if ($delay > 0 && $item->isHit()) { + $wait = (float) $item->get() + $delay - microtime(true); + if ($wait > 0) { + usleep((int) (min($wait, $delay) * 1_000_000)); + } + } + + $item->set(microtime(true)); + $item->expiresAfter(60); + $this->partInfoCache->save($item); + } + + public function searchByKeyword(string $keyword, array $options = []): array + { + $response = $this->request('/search', [ + 'query' => $keyword, + '[search_type]' => 'products', + ]); + + $content = $response->getContent(); + + //If we were redirected to a product page, then just return the single item + if (preg_match('#/product/(\d+)#', (string) $response->getInfo('url'), $matches)) { + return [$this->parseProductPage($content, $matches[1])]; + } + + $dom = new Crawler($content); + + $results = []; + + //Each result is a dt (picture and title) followed by a dd (text excerpt) + $dom->filter('dl#search > dt')->each(function (Crawler $node) use (&$results) { + $link = $node->filter('a[href*="/product/"]')->last(); + if ($link->count() === 0 || !preg_match('#/product/(\d+)#', $link->attr('href') ?? '', $matches)) { + return; + } + $id = $matches[1]; + + $image = $node->filter('img'); + $excerpt = $node->nextAll()->first(); + + $results[] = new SearchResultDTO( + provider_key: self::PROVIDER_KEY, + provider_id: $id, + //The title has the format "Pololu item 2130: DRV8833 Dual Motor Driver Carrier" + name: preg_replace('/^Pololu item\s*#?\s*\d+:\s*/', '', $link->text()), + description: $excerpt->count() > 0 && $excerpt->nodeName() === 'dd' ? $excerpt->text() : '', + preview_image_url: $image->count() > 0 ? $image->attr('src') : null, + provider_url: $this->getProductURL($id), + ); + }); + + return $results; + } + + public function getDetails(string $id, array $options = []): PartDetailDTO + { + //Ensure that $id is an item number + if (!ctype_digit($id)) { + throw new \InvalidArgumentException("The id must be a Pololu item number!"); + } + + $content = $this->request('/product/' . $id)->getContent(); + $dom = new Crawler($content); + + //The specifications and the files are on their own pages. The tabs of an empty page are not linked. + $specs = null; + if ($dom->filter('table.tabs a[href$="/specs"]')->count() > 0) { + $specs = $this->request('/product/' . $id . '/specs')->getContent(); + } + + $resources = null; + if ($dom->filter('table.tabs a[href$="/resources"]')->count() > 0) { + $resources = $this->request('/product/' . $id . '/resources')->getContent(); + } + + return $this->parseProductPage($content, $id, $specs, $resources); + } + + private function getProductURL(string $id): string + { + return self::BASE_URL . '/product/' . $id; + } + + /** + * @param string $content The HTML of the product page + * @param string $id The Pololu item number + * @param string|null $specs The HTML of the specs tab of the product, if available + * @param string|null $resources The HTML of the resources tab of the product, if available + */ + private function parseProductPage(string $content, string $id, ?string $specs = null, ?string $resources = null): PartDetailDTO + { + $dom = new Crawler($content); + + $productPageUrl = $this->getProductURL($id); + + $stock = $dom->filter('[data-available-stock]'); + + $purchaseInfo = new PurchaseInfoDTO( + distributor_name: self::DISTRIBUTOR_NAME, + order_number: $id, + prices: $this->parsePrices($dom), + product_url: $productPageUrl, + available_amount: $stock->count() > 0 && is_numeric($stock->attr('data-available-stock')) ? (float) $stock->attr('data-available-stock') : null, + ); + + $brand = $this->parseOrderFormValue($dom, 'Brand'); + //Pololu sells also products of other brands, for which the item number is no manufacturer part number + $isPololuProduct = $brand === null || strcasecmp($brand, 'Pololu') === 0; + + $images = $this->parseImages($dom); + $parameters = $specs !== null ? $this->parseParameters(new Crawler($specs)) : []; + + return new PartDetailDTO( + provider_key: self::PROVIDER_KEY, + provider_id: $id, + name: $dom->filter('h1#page_title')->text(), + description: $dom->filter('div#short_description')->text(''), + category: $this->parseCategory($dom), + manufacturer: $brand ?? 'Pololu', + mpn: $isPololuProduct ? $id : null, + preview_image_url: $images !== [] ? $images[0]->url : null, + manufacturing_status: $this->mapStatus($this->parseOrderFormValue($dom, 'Status')), + provider_url: $productPageUrl, + notes: $this->parseNotes($dom), + datasheets: $resources !== null ? $this->parseFiles(new Crawler($resources)) : [], + images: $images, + parameters: $parameters, + vendor_infos: [$purchaseInfo], + mass: $this->parseMass($parameters), + manufacturer_product_url: $isPololuProduct ? $productPageUrl : null, + ); + } + + /** + * Returns the value of a labeled field (like "Brand:" or "Status:") of the order form. + */ + private function parseOrderFormValue(Crawler $dom, string $label): ?string + { + $value = null; + + $dom->filter('span.label')->each(function (Crawler $node) use (&$value, $label) { + if ($value !== null || rtrim($this->cleanText($node->text()), ':') !== $label) { + return; + } + + $valueNode = $node->nextAll()->filter('span.value')->first(); + if ($valueNode->count() > 0) { + $value = $this->cleanText($valueNode->text()); + } + }); + + return $value === '' ? null : $value; + } + + /** + * Normalizes the whitespaces (including non-breaking spaces) of the given text + */ + private function cleanText(string $text): string + { + return trim(preg_replace('/[\s\x{00A0}]+/u', ' ', $text) ?? $text); + } + + private function mapStatus(?string $status): ManufacturingStatus + { + if ($status === null) { + return ManufacturingStatus::NOT_SET; + } + + $status = strtolower($status); + + return match (true) { + str_contains($status, 'discontinued'), str_contains($status, 'obsolete') => ManufacturingStatus::DISCONTINUED, + str_contains($status, 'end of life'), str_contains($status, 'last time buy') => ManufacturingStatus::EOL, + str_contains($status, 'not recommended') => ManufacturingStatus::NRFND, + str_contains($status, 'active') => ManufacturingStatus::ACTIVE, + default => ManufacturingStatus::NOT_SET, + }; + } + + private function parseCategory(Crawler $dom): ?string + { + $categories = $dom->filter('#breadcrumbs a')->each(fn(Crawler $node) => $this->cleanText($node->text())); + + return $categories === [] ? null : implode(' -> ', $categories); + } + + /** + * @return PriceDTO[] + */ + private function parsePrices(Crawler $dom): array + { + $prices = []; + + //Each row of the table consists of the minimum order amount and the unit price + $dom->filter('table.pricing tr')->each(function (Crawler $row) use (&$prices) { + $cells = $row->filter('td'); + if ($cells->count() < 2) { + return; + } + + $amount = str_replace(',', '', $this->cleanText($cells->eq(0)->text())); + //If the product is on sale, the cell contains the regular price followed by the reduced one + if (!is_numeric($amount) || !preg_match_all('/\d+(?:\.\d+)?/', str_replace(',', '', $cells->eq(1)->text()), $matches)) { + return; + } + + $prices[] = new PriceDTO( + minimum_discount_amount: (float) $amount, + price: end($matches[0]), + currency_iso_code: self::CURRENCY, + includes_tax: false, + ); + }); + + return $prices; + } + + /** + * @return FileDTO[] + */ + private function parseImages(Crawler $dom): array + { + $images = []; + + //The main picture carries the list of all pictures of the product for the gallery + $main = $dom->filter('#main_picture_img'); + if ($main->count() > 0) { + $gallery = json_decode($main->attr('data-gallery-pictures') ?? '', true); + foreach (is_array($gallery) ? $gallery : [] as $picture) { + $url = $picture['url_full'] ?? $picture['url_medium'] ?? null; + if (!is_string($url) || isset($images[$url])) { + continue; + } + + $caption = $this->cleanText(html_entity_decode(strip_tags((string) ($picture['caption'] ?? '')), ENT_QUOTES | ENT_HTML5)); + $images[$url] = new FileDTO($url, $caption !== '' ? $caption : null); + } + } + + if ($images === []) { + $ogImage = $dom->filter('meta[property="og:image"]'); + if ($ogImage->count() > 0 && $ogImage->attr('content')) { + return [new FileDTO($ogImage->attr('content'))]; + } + } + + return array_values($images); + } + + private function parseNotes(Crawler $dom): string + { + $tabPage = $dom->filter('div.tab_page'); + if ($tabPage->count() === 0) { + return ''; + } + + /** @var \DOMElement $root */ + $root = $tabPage->getNode(0)->cloneNode(true); + + //Remove the pictures (they are returned as images), scripts and embedded videos + $remove = []; + foreach (['table', 'script', 'style', 'iframe', 'img'] as $tag) { + foreach ($root->getElementsByTagName($tag) as $element) { + if ($tag !== 'table' || str_contains($element->getAttribute('class'), 'picture_with_caption')) { + $remove[] = $element; + } + } + } + foreach ($remove as $element) { + $element->parentNode?->removeChild($element); + } + + //Make the links absolute, so they work outside of pololu.com + foreach ($root->getElementsByTagName('a') as $link) { + $link->removeAttribute('onclick'); + if ($link->hasAttribute('href')) { + $link->setAttribute('href', $this->absoluteURL($link->getAttribute('href'))); + } + } + + $html = ''; + foreach ($root->childNodes as $child) { + $html .= $root->ownerDocument->saveHTML($child); + } + + //The removed pictures leave empty paragraphs behind + return trim(preg_replace('#

\s*

#', '', $html) ?? $html); + } + + private function absoluteURL(string $url): string + { + if (str_starts_with($url, '//')) { + return 'https:' . $url; + } + if (str_starts_with($url, '/')) { + return self::BASE_URL . $url; + } + + return $url; + } + + /** + * Parses the specs tab of a product: Each table of specifications is preceded by a heading, which is used as group. + * @return ParameterDTO[] + */ + private function parseParameters(Crawler $dom): array + { + $parameters = []; + $group = null; + + $dom->filter('div.tab_page')->children()->each(function (Crawler $node) use (&$parameters, &$group) { + if ($node->nodeName() === 'h2') { + $group = $this->cleanText($node->text()); + return; + } + + if ($node->nodeName() !== 'table' || !str_contains($node->attr('class') ?? '', 'specifications')) { + return; + } + + $node->filter('tr')->each(function (Crawler $row) use (&$parameters, $group) { + $header = $row->filter('th'); + $cell = $row->filter('td'); + if ($header->count() === 0 || $cell->count() === 0) { + return; + } + + $name = rtrim($this->cleanText($header->text()), ':'); + $value = $this->textWithoutFootnotes($cell); + if ($name === '' || $value === '') { + return; + } + + //Yes/no specifications are named like a question ("Reverse voltage protection?") and answered with Y or N + if (str_ends_with($name, '?')) { + $value = match ($value) { + 'Y' => 'Yes', + 'N' => 'No', + default => $value, + }; + } + + //Only split values like "2.7 V" into number and unit, but not markings like "0J3762" + if (preg_match('/^-?\d+(?:\.\d+)?\s*[^\d\s]*$/u', $value)) { + $parameters[] = ParameterDTO::parseValueIncludingUnit(name: $name, value: $value, group: $group); + } else { + $parameters[] = new ParameterDTO(name: $name, value_text: $value, group: $group); + } + }); + }); + + return $parameters; + } + + /** + * Returns the text of the given node without the footnote references (like the "1" in "1.0 g¹") + */ + private function textWithoutFootnotes(Crawler $node): string + { + /** @var \DOMElement $element */ + $element = $node->getNode(0)->cloneNode(true); + + foreach (iterator_to_array($element->getElementsByTagName('sup')) as $footnote) { + $footnote->parentNode?->removeChild($footnote); + } + + return $this->cleanText($element->textContent); + } + + /** + * Returns the weight in grams, if the specifications contain it + * @param ParameterDTO[] $parameters + */ + private function parseMass(array $parameters): ?float + { + foreach ($parameters as $parameter) { + if (strcasecmp($parameter->name, 'Weight') !== 0 || $parameter->value_typ === null) { + continue; + } + + return match ($parameter->unit) { + 'g' => $parameter->value_typ, + 'kg' => $parameter->value_typ * 1000, + 'mg' => $parameter->value_typ / 1000, + 'oz' => round($parameter->value_typ * 28.3495, 2), + 'lb', 'lbs' => round($parameter->value_typ * 453.592, 2), + default => null, + }; + } + + return null; + } + + /** + * Parses the resources tab of a product. All file downloads (datasheets, dimension diagrams, 3D models, ...) are + * returned, of the other resources (recommended links, guides, ...) only the ones which link directly to a file. + * @return FileDTO[] + */ + private function parseFiles(Crawler $dom): array + { + $files = []; + + $dom->filter('ul.resources')->each(function (Crawler $list) use (&$files) { + $isFileList = str_contains($list->attr('class') ?? '', 'attached_files'); + + $list->filter('li.resource .content_title a[href]')->each(function (Crawler $link) use (&$files, $isFileList) { + $url = $this->absoluteURL($link->attr('href') ?? ''); + if (!str_starts_with($url, 'http') || isset($files[$url])) { + return; + } + + if (!$isFileList && !preg_match(self::FILE_EXTENSIONS_REGEX, (string) parse_url($url, PHP_URL_PATH))) { + return; + } + + $files[$url] = new FileDTO($url, $this->cleanText($link->text())); + }); + }); + + return array_values($files); + } + + public function getHandledDomains(): array + { + return ['pololu.com']; + } + + public function getIDFromURL(string $url): ?string + { + //URL like: https://www.pololu.com/product/2130 or https://www.pololu.com/product/2130/resources + $matches = []; + if (preg_match('#/product/(\d+)(?:[/?\#]|$)#', $url, $matches)) { + return $matches[1]; + } + + return null; + } +} diff --git a/src/Settings/InfoProviderSystem/InfoProviderSettings.php b/src/Settings/InfoProviderSystem/InfoProviderSettings.php index 519f95895..221b2a88d 100644 --- a/src/Settings/InfoProviderSystem/InfoProviderSettings.php +++ b/src/Settings/InfoProviderSystem/InfoProviderSettings.php @@ -87,4 +87,6 @@ class InfoProviderSettings #[EmbeddedSettings] public ?TrustedPartsSettings $trustedparts = null; + #[EmbeddedSettings] + public ?PololuSettings $pololu = null; } diff --git a/src/Settings/InfoProviderSystem/PololuSettings.php b/src/Settings/InfoProviderSystem/PololuSettings.php new file mode 100644 index 000000000..a19a5a30a --- /dev/null +++ b/src/Settings/InfoProviderSystem/PololuSettings.php @@ -0,0 +1,51 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Settings\InfoProviderSystem; + +use App\Settings\SettingsIcon; +use Jbtronics\SettingsBundle\Metadata\EnvVarMode; +use Jbtronics\SettingsBundle\Settings\Settings; +use Jbtronics\SettingsBundle\Settings\SettingsParameter; +use Jbtronics\SettingsBundle\Settings\SettingsTrait; +use Symfony\Component\Form\Extension\Core\Type\NumberType; +use Symfony\Component\Translation\TranslatableMessage as TM; +use Symfony\Component\Validator\Constraints as Assert; + +#[Settings(label: new TM("settings.ips.pololu"), description: new TM("settings.ips.pololu.help"))] +#[SettingsIcon("fa-plug")] +class PololuSettings +{ + use SettingsTrait; + + #[SettingsParameter(label: new TM("settings.ips.lcsc.enabled"), + envVar: "bool:PROVIDER_POLOLU_ENABLED", envVarMode: EnvVarMode::OVERWRITE)] + public bool $enabled = false; + + /** @var int The minimum time in seconds between two page requests to pololu.com */ + #[SettingsParameter(label: new TM("settings.ips.pololu.requestDelay"), description: new TM("settings.ips.pololu.requestDelay.help"), + formType: NumberType::class, formOptions: ["attr" => ["min" => 0, "max" => 30]], + envVar: "int:PROVIDER_POLOLU_REQUEST_DELAY", envVarMode: EnvVarMode::OVERWRITE)] + #[Assert\Range(min: 0, max: 30)] + public int $requestDelay = 1; +} diff --git a/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php b/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php new file mode 100644 index 000000000..151a451a6 --- /dev/null +++ b/tests/Services/InfoProviderSystem/Providers/PololuProviderTest.php @@ -0,0 +1,367 @@ +. + */ + +declare(strict_types=1); + +namespace App\Tests\Services\InfoProviderSystem\Providers; + +use App\Entity\Parts\ManufacturingStatus; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use App\Services\InfoProviderSystem\Providers\PololuProvider; +use App\Services\InfoProviderSystem\Providers\ProviderCapabilities; +use App\Settings\InfoProviderSystem\PololuSettings; +use App\Tests\SettingsTestHelper; +use PHPUnit\Framework\TestCase; +use Symfony\Component\Cache\Adapter\ArrayAdapter; +use Symfony\Component\HttpClient\MockHttpClient; +use Symfony\Component\HttpClient\Response\MockResponse; + +final class PololuProviderTest extends TestCase +{ + private PololuSettings $settings; + private PololuProvider $provider; + private MockHttpClient $httpClient; + + private const SEARCH_PAGE = <<<'HTML' +
+ +
+ HTML; + + private const PRODUCT_PAGE = <<<'HTML' + + +
+ +

DRV8833 Dual Motor Driver Carrier

+
+ + +
+ + + + + +
Pololu item #: 2130
Brand: Pololu
Status: Active and Preferred 
+ + + + + +
Price breakUnit price (US$)
+ 1 + + 10.95 +
510.07
1008.53
+
+
+

This tiny breakout board for TI’s DRV8833 dual motor driver can deliver 1.2 A per channel.

+
+ %TABS%
Description
+

Overview

+

Bottom view with dimensions.

+

We also carry a DRV8835 dual motor driver carrier.

+
+
+ HTML; + + private const TABS = <<<'HTML' + Specs (5) + Resources (4) + HTML; + + private const EMPTY_TABS = <<<'HTML' + Specs (0) + Resources (0) + HTML; + + private const SPECS_PAGE = <<<'HTML' +

Dimensions

+ + + +
+ Size: + + 0.5″ × 0.8″1 +
Weight:1.0 g1
+

General specifications

+ + + + + +
Motor driver:DRV8833
Minimum operating voltage:2.7 V
Reverse voltage protection?:Y
Other PCB markings:0J3762
+

Notes:

1
Without included hardware.
+
+ HTML; + + private const RESOURCES_PAGE = <<<'HTML' +

File downloads

+ +

Recommended links

+ +
+ HTML; + + protected function setUp(): void + { + $this->httpClient = new MockHttpClient(); + $this->settings = SettingsTestHelper::createSettingsDummy(PololuSettings::class); + $this->settings->enabled = true; + //Do not slow down the tests + $this->settings->requestDelay = 0; + $this->provider = new PololuProvider($this->httpClient, $this->settings, new ArrayAdapter()); + } + + private function productPage(string $tabs = self::TABS): string + { + return str_replace('%TABS%', $tabs, self::PRODUCT_PAGE); + } + + public function testGetProviderInfo(): void + { + $info = $this->provider->getProviderInfo(); + + $this->assertSame('pololu', $info->key); + $this->assertSame('Pololu', $info->name); + $this->assertSame(PololuSettings::class, $info->settingsClass); + $this->assertContains(ProviderCapabilities::BASIC, $info->capabilities); + $this->assertContains(ProviderCapabilities::PICTURE, $info->capabilities); + $this->assertContains(ProviderCapabilities::DATASHEET, $info->capabilities); + $this->assertContains(ProviderCapabilities::PRICE, $info->capabilities); + $this->assertContains(ProviderCapabilities::PARAMETERS, $info->capabilities); + } + + public function testIsActive(): void + { + $this->assertTrue($this->provider->isActive()); + + $this->settings->enabled = false; + $this->assertFalse($this->provider->isActive()); + } + + public function testSearchByKeyword(): void + { + $requestedUrl = null; + $this->httpClient->setResponseFactory(function (string $method, string $url) use (&$requestedUrl) { + $requestedUrl = $url; + return new MockResponse(self::SEARCH_PAGE); + }); + + $results = $this->provider->searchByKeyword('DRV8833'); + + $this->assertStringStartsWith('https://www.pololu.com/search?', $requestedUrl); + $this->assertStringContainsString('query=DRV8833', $requestedUrl); + $this->assertStringContainsString('search_type]=products', $requestedUrl); + + //The category result must be skipped + $this->assertCount(2, $results); + $this->assertContainsOnlyInstancesOf(SearchResultDTO::class, $results); + + $this->assertSame('pololu', $results[0]->provider_key); + $this->assertSame('2130', $results[0]->provider_id); + $this->assertSame('DRV8833 Dual Motor Driver Carrier', $results[0]->name); + $this->assertSame('https://www.pololu.com/product/2130', $results[0]->provider_url); + $this->assertSame('https://a.pololu-files.com/picture/0J3864.100.jpg?abc', $results[0]->preview_image_url); + $this->assertStringContainsString('carrier for the DRV8833', $results[0]->description); + + $this->assertSame('2196', $results[1]->provider_id); + $this->assertSame('Tamiya 70203 Low-Current Motor Gearbox (3-Speed)', $results[1]->name); + } + + public function testSearchByKeywordWithoutResults(): void + { + $this->httpClient->setResponseFactory([new MockResponse('')]); + + $this->assertSame([], $this->provider->searchByKeyword('doesnotexist')); + } + + public function testGetDetails(): void + { + $requestedUrls = []; + $responses = [$this->productPage(), self::SPECS_PAGE, self::RESOURCES_PAGE]; + $this->httpClient->setResponseFactory(function (string $method, string $url) use (&$requestedUrls, &$responses) { + $requestedUrls[] = $url; + return new MockResponse(array_shift($responses)); + }); + + $details = $this->provider->getDetails('2130'); + + $this->assertSame([ + 'https://www.pololu.com/product/2130', + 'https://www.pololu.com/product/2130/specs', + 'https://www.pololu.com/product/2130/resources', + ], $requestedUrls); + + $this->assertInstanceOf(PartDetailDTO::class, $details); + $this->assertSame('pololu', $details->provider_key); + $this->assertSame('2130', $details->provider_id); + $this->assertSame('DRV8833 Dual Motor Driver Carrier', $details->name); + $this->assertStringStartsWith('This tiny breakout board', $details->description); + $this->assertSame('Electronics -> Brushed DC Motor Drivers', $details->category); + $this->assertSame('Pololu', $details->manufacturer); + $this->assertSame('2130', $details->mpn); + $this->assertSame(ManufacturingStatus::ACTIVE, $details->manufacturing_status); + $this->assertSame('https://www.pololu.com/product/2130', $details->provider_url); + $this->assertSame('https://www.pololu.com/product/2130', $details->manufacturer_product_url); + $this->assertSame(1.0, $details->mass); + + //Images + $this->assertSame('https://a.pololu-files.com/picture/0J3864.1200.jpg?abc', $details->preview_image_url); + $this->assertCount(2, $details->images); + $this->assertSame('https://a.pololu-files.com/picture/0J3864.1200.jpg?abc', $details->images[0]->url); + $this->assertSame('DRV8833 dual motor driver carrier.', $details->images[0]->name); + $this->assertSame('https://a.pololu-files.com/picture/0J3872.1200.png?def', $details->images[1]->url); + + //Notes: pictures are removed and links are made absolute + $this->assertStringContainsString('

Overview

', $details->notes); + $this->assertStringContainsString('href="https://www.pololu.com/product/2135"', $details->notes); + $this->assertStringNotContainsString('picture_with_caption', $details->notes); + $this->assertStringNotContainsString('onclick', $details->notes); + + //Prices and stock + $this->assertCount(1, $details->vendor_infos); + $purchaseInfo = $details->vendor_infos[0]; + $this->assertSame('Pololu', $purchaseInfo->distributor_name); + $this->assertSame('2130', $purchaseInfo->order_number); + $this->assertSame('https://www.pololu.com/product/2130', $purchaseInfo->product_url); + $this->assertSame(148.0, $purchaseInfo->available_amount); + $this->assertFalse($purchaseInfo->prices_include_vat); + $this->assertCount(3, $purchaseInfo->prices); + $this->assertSame(1.0, $purchaseInfo->prices[0]->minimum_discount_amount); + $this->assertSame('10.95', $purchaseInfo->prices[0]->price); + $this->assertSame('USD', $purchaseInfo->prices[0]->currency_iso_code); + $this->assertSame(100.0, $purchaseInfo->prices[2]->minimum_discount_amount); + $this->assertSame('8.53', $purchaseInfo->prices[2]->price); + + //Files: all file downloads, but only the recommended links which point to a file + $this->assertCount(3, $details->datasheets); + $this->assertSame('https://www.pololu.com/file/0J1615/drv8833-dual-motor-driver-carrier-dimensions.pdf', $details->datasheets[0]->url); + $this->assertSame('Dimension diagram of the DRV8833 Dual Motor Driver Carrier', $details->datasheets[0]->name); + $this->assertSame('https://www.pololu.com/file/0J1616/drv8833-dual-motor-driver-carrier-step.step', $details->datasheets[1]->url); + $this->assertSame('https://www.ti.com/lit/ds/symlink/drv8833.pdf', $details->datasheets[2]->url); + + //Parameters + $this->assertCount(6, $details->parameters); + $parameters = []; + foreach ($details->parameters as $parameter) { + $parameters[$parameter->name] = $parameter; + } + + $this->assertSame('0.5″ × 0.8″', $parameters['Size']->value_text); + $this->assertSame('Dimensions', $parameters['Size']->group); + $this->assertSame(1.0, $parameters['Weight']->value_typ); + $this->assertSame('g', $parameters['Weight']->unit); + $this->assertSame('DRV8833', $parameters['Motor driver']->value_text); + $this->assertSame('General specifications', $parameters['Motor driver']->group); + $this->assertSame(2.7, $parameters['Minimum operating voltage']->value_typ); + $this->assertSame('V', $parameters['Minimum operating voltage']->unit); + $this->assertSame('Yes', $parameters['Reverse voltage protection?']->value_text); + //A marking must not be split into a number and a unit + $this->assertSame('0J3762', $parameters['Other PCB markings']->value_text); + $this->assertNull($parameters['Other PCB markings']->value_typ); + } + + public function testGetDetailsSkipsEmptyTabs(): void + { + $requests = 0; + $this->httpClient->setResponseFactory(function () use (&$requests) { + $requests++; + return new MockResponse($this->productPage(self::EMPTY_TABS)); + }); + + $details = $this->provider->getDetails('2130'); + + $this->assertSame(1, $requests); + $this->assertSame([], $details->parameters); + $this->assertSame([], $details->datasheets); + $this->assertNull($details->mass); + } + + public function testGetDetailsOfOtherBrand(): void + { + $page = str_replace('Pololu', 'Tamiya', $this->productPage(self::EMPTY_TABS)); + $this->httpClient->setResponseFactory([new MockResponse($page)]); + + $details = $this->provider->getDetails('2196'); + + $this->assertSame('Tamiya', $details->manufacturer); + //The Pololu item number is no manufacturer part number for products of other brands + $this->assertNull($details->mpn); + $this->assertNull($details->manufacturer_product_url); + $this->assertSame('2196', $details->vendor_infos[0]->order_number); + } + + public function testGetDetailsWithInvalidId(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->provider->getDetails('abc'); + } + + public function testNoFurtherRequestsAfterBeingBlocked(): void + { + $requests = 0; + $this->httpClient->setResponseFactory(function () use (&$requests) { + $requests++; + return new MockResponse('Too many requests', ['http_code' => 429]); + }); + + try { + $this->provider->searchByKeyword('DRV8833'); + $this->fail('An exception should have been thrown'); + } catch (\RuntimeException $exception) { + $this->assertStringContainsString('429', $exception->getMessage()); + } + + try { + $this->provider->getDetails('2130'); + $this->fail('An exception should have been thrown'); + } catch (\RuntimeException) { + //The second lookup must not have sent a request + $this->assertSame(1, $requests); + } + } + + public function testGetHandledDomains(): void + { + $this->assertSame(['pololu.com'], $this->provider->getHandledDomains()); + } + + public function testGetIDFromURL(): void + { + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130')); + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130/resources')); + $this->assertSame('2130', $this->provider->getIDFromURL('https://www.pololu.com/product/2130?foo=bar')); + $this->assertNull($this->provider->getIDFromURL('https://www.pololu.com/category/11/brushed-dc-motor-drivers')); + } +} diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 87f599162..313ccbce4 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -15686,5 +15686,29 @@ Buerklin-API Authentication server: Available stock + + + settings.ips.pololu + Pololu + + + + + settings.ips.pololu.help + Pololu.com offers no official API, so this info provider webscrapes the website to extract info. It could break at any time, use it at your own risk. + + + + + settings.ips.pololu.requestDelay + Delay between page requests + + + + + settings.ips.pololu.requestDelay.help + The minimum time in seconds between two page requests to pololu.com. Getting the details of a product needs up to three page requests. + +