From ae6c2ad81d1d20787c44d75030901f369345b0fd Mon Sep 17 00:00:00 2001 From: Justin Sherrill Date: Tue, 26 May 2020 13:39:32 -0400 Subject: [PATCH 1/4] Fixes #29933 - log error when repo scan errors with an unexpected exception (cherry picked from commit 394c8b6a3daaf59bb8942b0f555f030a4a107a52) --- app/lib/katello/util/cdn_var_substitutor.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/lib/katello/util/cdn_var_substitutor.rb b/app/lib/katello/util/cdn_var_substitutor.rb index 228f583e916..4dd3251b16c 100644 --- a/app/lib/katello/util/cdn_var_substitutor.rb +++ b/app/lib/katello/util/cdn_var_substitutor.rb @@ -67,6 +67,7 @@ def find_substitutions(paths_with_substitutions) futures.each do |future| resolved << future.value + Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected? end find_substitutions(resolved.compact.flatten) From a10d6a8b91a5119f7dc62bdc8d2c000f12701df3 Mon Sep 17 00:00:00 2001 From: Ian Ballou Date: Mon, 28 Sep 2020 10:25:49 -0400 Subject: [PATCH 2/4] Fixes #30775 - publishing large cv w/ filters fails Pulp 3 yum (#8935) * Fixes #30775 - publishing large cv w/ filters fails Pulp 3 yum * Refs #30775 - refactor small-repo chunk skipping and more chunked copy tests added (cherry picked from commit b64f6f7b8e441a1b16c10a06acafc1c141fa538c) --- .../pulp3/repository/multi_copy_content.rb | 2 +- app/services/katello/pulp3/repository/yum.rb | 64 ++++++++- .../pulp3/repository/yum/copy_units_test.rb | 123 ++++++++++++++++++ 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 test/services/katello/pulp3/repository/yum/copy_units_test.rb diff --git a/app/lib/actions/pulp3/repository/multi_copy_content.rb b/app/lib/actions/pulp3/repository/multi_copy_content.rb index 0d1c9b15daf..40530116ca9 100644 --- a/app/lib/actions/pulp3/repository/multi_copy_content.rb +++ b/app/lib/actions/pulp3/repository/multi_copy_content.rb @@ -17,7 +17,7 @@ def invoke_external_task repo_id_map = {} input[:repo_id_map].each do |source_repo_ids, dest_repo_map| - repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map + repo_id_map[JSON.parse(source_repo_ids)] = dest_repo_map.deep_dup end output[:pulp_tasks] = ::Katello::Repository.find(repo_id_map.values.first[:dest_repo]).backend_service(smart_proxy).copy_content_from_mapping(repo_id_map, input) diff --git a/app/services/katello/pulp3/repository/yum.rb b/app/services/katello/pulp3/repository/yum.rb index af534dc714c..939239bff10 100644 --- a/app/services/katello/pulp3/repository/yum.rb +++ b/app/services/katello/pulp3/repository/yum.rb @@ -3,10 +3,13 @@ module Katello module Pulp3 class Repository + # rubocop:disable Metrics/ClassLength class Yum < ::Katello::Pulp3::Repository include Katello::Util::Errata include Katello::Util::PulpcoreContentFilters + UNIT_LIMIT = 10_000 + def remote_options if root.url.blank? common_remote_options.merge(url: nil, policy: root.download_policy) @@ -104,14 +107,71 @@ def multi_copy_units(repo_id_map, dependency_solving) data.config << config end end - # FIXME: data's content being [] causes all content to be copied back - tasks << api.copy_api.copy_content(data) + tasks << copy_content_chunked(data) else tasks << remove_all_content_from_mapping(repo_id_map) end tasks.flatten end + def copy_api_data_dup(data) + data_dup = PulpRpmClient::Copy.new + data_dup.dependency_solving = data.dependency_solving + data_dup.config = [] + data.config.each do |repo_config| + config_hash = { + source_repo_version: repo_config[:source_repo_version], + dest_repo: repo_config[:dest_repo], + content: [] + } + config_hash[:dest_base_version] = repo_config[:dest_base_version] if repo_config[:dest_base_version] + data_dup.config << config_hash + end + data_dup + end + + def copy_content_chunked(data) + tasks = [] + # Don't chunk if there aren't enough content units + if data.config.sum { |repo_config| repo_config[:content].size } <= UNIT_LIMIT + return api.copy_api.copy_content(data) + end + + unit_copy_counter = 0 + i = 0 + leftover_units = data.config.first[:content].deep_dup + + # Copy data and clear its content fields + data_dup = copy_api_data_dup(data) + + while i < data_dup.config.size + # Copy all units within repo or only some? + if leftover_units.length < UNIT_LIMIT - unit_copy_counter + copy_amount = leftover_units.length + else + copy_amount = UNIT_LIMIT - unit_copy_counter + end + + data_dup.config[i][:content] = leftover_units.pop(copy_amount) + unit_copy_counter += copy_amount + # Do copy call if limit is reached or if we're under the limit but on the last repo config. + if unit_copy_counter >= UNIT_LIMIT || (i == data_dup.config.size - 1 && leftover_units.empty?) + tasks << api.copy_api.copy_content(data_dup) + unit_copy_counter = 0 + end + + if leftover_units.empty? + # Nothing more to copy -- clear current config's content + data_dup.config[i][:content] = [] + i += 1 + # Fetch unit list for next data config + leftover_units = data.config[i][:content].deep_dup unless i == data_dup.config.size + end + end + + tasks + end + def remove_all_content_from_mapping(repo_id_map) tasks = [] repo_id_map.each do |_source_repo_ids, dest_repo_id_map| diff --git a/test/services/katello/pulp3/repository/yum/copy_units_test.rb b/test/services/katello/pulp3/repository/yum/copy_units_test.rb new file mode 100644 index 00000000000..1ae3e8d4ed8 --- /dev/null +++ b/test/services/katello/pulp3/repository/yum/copy_units_test.rb @@ -0,0 +1,123 @@ +require 'katello_test_helper' + +module Katello + module Service + class Repository + class YumCopyUnitsTest < ::ActiveSupport::TestCase + # rubocop:disable Metrics/MethodLength + include RepositorySupport + + def setup + @mock_smart_proxy = mock('smart_proxy') + @mock_smart_proxy.stubs(:pulp3_support?).returns(true) + @mock_smart_proxy.stubs(:pulp2_preferred_for_type?).returns(false) + @mock_smart_proxy.stubs(:pulp_master?).returns(true) + @repo = katello_repositories(:fedora_17_x86_64_duplicate) + @repo_service = @repo.backend_service(@mock_smart_proxy) + end + + def test_copy_api_data_dup_does_deep_copy + data = PulpRpmClient::Copy.new + data.config = [ + { source_repo_version: "a source repo", + dest_repo: "a dest repo", + content: ["1", "2", "3"], + dest_base_version: 0 }, + { source_repo_version: "another source repo", + dest_repo: "another dest repo", + content: ["4", "5", "6"], + dest_base_version: 1 } + ] + data.dependency_solving = false + + data_dup = @repo_service.copy_api_data_dup(data) + + refute data.equal?(data_dup) + end + + def test_copy_api_data_dup_clears_content + data = PulpRpmClient::Copy.new + data.config = [ + { source_repo_version: "a source repo", + dest_repo: "a dest repo", + content: ["1", "2", "3"], + dest_base_version: 0 }, + { source_repo_version: "another source repo", + dest_repo: "another dest repo", + content: ["4", "5", "6"], + dest_base_version: 1 } + ] + data.dependency_solving = false + + data.config.first[:content] = [] + data.config.second[:content] = [] + + data_dup = @repo_service.copy_api_data_dup(data) + + assert_equal data, data_dup + end + + def test_copy_content_chunked_limits_units_copied + data = PulpRpmClient::Copy.new + data.config = [] + + content = [] + 30_001.times { |i| content << i } + + 3.times { data.config << { content: content } } + + mock_api = "test" + Katello::Pulp3::Api::Yum.any_instance.expects(:copy_api).returns(mock_api).times(10) + mock_api.stubs(:copy_content).returns("copied") + + @repo_service.copy_content_chunked(data) + end + + def test_copy_content_chunked_copies_correct_units + data = PulpRpmClient::Copy.new + data.config = [] + + mock_api = "test" + Katello::Pulp3::Api::Yum.any_instance.expects(:copy_api).returns(mock_api).times(4) + + 3.times do + data.config << { + source_repo_version: "repo version", + dest_repo: "dest repo", + content: [] + } + end + data.config[0][:content] = (0..9_999).to_a + data.config[1][:content] = (10_000..19_999).to_a + data.config[2][:content] = (20_000..30_000).to_a + + mock_api.expects(:copy_content).returns("task").once.with do |value| + value.config == [{source_repo_version: "repo version", dest_repo: "dest repo", content: (0..9_999).to_a}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: []}] + end + + mock_api.expects(:copy_content).returns("task").once.with do |value| + value.config == [{source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: (10_000..19_999).to_a}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: []}] + end + + mock_api.expects(:copy_content).returns("task").once.with do |value| + value.config == [{source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: (20_001..30_000).to_a}] + end + + mock_api.expects(:copy_content).returns("task").once.with do |value| + value.config == [{source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: []}, + {source_repo_version: "repo version", dest_repo: "dest repo", content: [20_000]}] + end + + @repo_service.copy_content_chunked(data) + end + end + end + end +end From 03d7ad06627e528040ce2463e5b21cb975e1b1e6 Mon Sep 17 00:00:00 2001 From: Justin Sherrill Date: Tue, 22 Sep 2020 09:12:19 -0400 Subject: [PATCH 3/4] Fixes #30694 - limit threading on cdn scan (cherry picked from commit e433f4a4dca2ee7168002d7e76cbc10a0b0054b5) --- app/lib/katello/resources/cdn.rb | 5 +++-- app/lib/katello/util/cdn_var_substitutor.rb | 16 +++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/lib/katello/resources/cdn.rb b/app/lib/katello/resources/cdn.rb index abdd4a3ff67..76fce34ae6b 100644 --- a/app/lib/katello/resources/cdn.rb +++ b/app/lib/katello/resources/cdn.rb @@ -18,7 +18,7 @@ def self.parse_version(releasever) class CdnResource CDN_DOCKER_CONTAINER_LISTING = "CONTAINER_REGISTRY_LISTING".freeze - attr_reader :url, :product, :options + attr_reader :url, :product, :options, :proxy def substitutor(logger = nil) @logger = logger @@ -26,6 +26,7 @@ def substitutor(logger = nil) end def initialize(url, options = {}) + @proxy = ::HttpProxy.default_global_content_proxy @ssl_version = Setting[:cdn_ssl_version] if @ssl_version && !SUPPORTED_SSL_VERSIONS.include?(@ssl_version) fail("Invalid SSL version specified. Check the 'CDN SSL Version' setting") @@ -140,7 +141,7 @@ def self.ca_file_contents end def net_http_class - if (proxy = ::HttpProxy.default_global_content_proxy) + if self.proxy uri = URI(proxy.url) #Net::HTTP::Proxy ignores port as part of the url Net::HTTP::Proxy("#{uri.host}#{uri.path}", uri.port, proxy.username, proxy.password) else diff --git a/app/lib/katello/util/cdn_var_substitutor.rb b/app/lib/katello/util/cdn_var_substitutor.rb index 4dd3251b16c..f57231bb0d2 100644 --- a/app/lib/katello/util/cdn_var_substitutor.rb +++ b/app/lib/katello/util/cdn_var_substitutor.rb @@ -59,15 +59,17 @@ def find_substitutions(paths_with_substitutions) return resolved if to_resolve.empty? - futures = to_resolve.map do |path_with_substitution| - Concurrent::Promises.future do - path_with_substitution.resolve_substitutions(@resource) + to_resolve.in_groups_of(8) do |group| + futures = group.map do |path_with_substitution| + Concurrent::Promises.future do + path_with_substitution.resolve_substitutions(@resource) + end end - end - futures.each do |future| - resolved << future.value - Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected? + futures.each do |future| + resolved << future.value + Rails.logger.error("Failed at scanning for repository: #{future.reason}") if future.rejected? + end end find_substitutions(resolved.compact.flatten) From 0a2d4ae8dfeb911ce5949c0021b203cc13443554 Mon Sep 17 00:00:00 2001 From: ianballou Date: Mon, 28 Sep 2020 15:32:24 -0400 Subject: [PATCH 4/4] Update changelog and version for Katello 3.16.1 --- CHANGELOG.md | 43 ++++++++++++++++++++++++++++++++++++++++++ lib/katello/version.rb | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5f2575b10..0377c8e9a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -231,3 +231,46 @@ * Error while running katello:pulp3_migration ([#30472](https://projects.theforeman.org/issues/30472)) * podman search against katello ([#29742](https://projects.theforeman.org/issues/29742), [8c5c4833](https://github.com/Katello/katello.git/commit/8c5c483357d2340c84f38cbfb0e42ee04bb4f8be)) * Database migration fails on SQLite ([#29549](https://projects.theforeman.org/issues/29549), [fc1dbc74](https://github.com/Katello/katello.git/commit/fc1dbc745a1b7df2d657b70bf2a58b8de2235341)) + +# 3.16.1 Tasty Taiyaki (2020-09-28) + +## Features + +### Repositories + * Support verify checksum scan in pulp3 ([#30190](https://projects.theforeman.org/issues/30190), [a2e8304e](https://github.com/Katello/katello.git/commit/a2e8304ed638bdc81e4fdb93dd8ef4b175e8b456)) + * Support 'Skip metadata check' form of syncing with pulp3 (optimize=False) with pulp3 ([#30189](https://projects.theforeman.org/issues/30189), [ee06f150](https://github.com/Katello/katello.git/commit/ee06f1508165ec2be53324361882e876a211bffc)) + +### Other + * Expose sortable_version as template macro ([#30561](https://projects.theforeman.org/issues/30561), [dc91a5a2](https://github.com/Katello/katello.git/commit/dc91a5a2f1903e434e72c63648f84b00d42aff20), [8a69eadb](https://github.com/Katello/katello.git/commit/8a69eadbf1447b2f4ffd257096eea9a54b888d20)) + +## Bug Fixes + +### Content Views + * publishing a large content view with filters fails with error on multi-copy api ([#30775](https://projects.theforeman.org/issues/30775), [b64f6f7b](https://github.com/Katello/katello.git/commit/b64f6f7b8e441a1b16c10a06acafc1c141fa538c)) + * hammer content-view version incremental-update fails with ISE ([#30725](https://projects.theforeman.org/issues/30725), [902f0789](https://github.com/Katello/katello.git/commit/902f07899002c39fbc078d96416c094195b3ffb7)) + * Content view publish fails if repo checksum type changes ([#30641](https://projects.theforeman.org/issues/30641), [7b59d75e](https://github.com/Katello/katello.git/commit/7b59d75e9c70002dbef6387e9b01ad62d0af02ea)) + * 'include packages with no errata' package filter does not work with pulp3 ([#30533](https://projects.theforeman.org/issues/30533), [026fd8de](https://github.com/Katello/katello.git/commit/026fd8dee9a245a28c4e0bdb081960f3b55d2e43)) + +### Repositories + * Error Row was updated or deleted by another transaction when deleting docker repository ([#30766](https://projects.theforeman.org/issues/30766), [26eb0351](https://github.com/Katello/katello.git/commit/26eb03518f8ce13bd0247b5fc63f7830d25e3d6f)) + * cannot communicate with pulpcore if foreman_azure_rm is installed "no implicit conversion of OpenSSL::X509::Certificate into String (TypeError) (Faraday::ConnectionFailed)" ([#30746](https://projects.theforeman.org/issues/30746), [d2b75839](https://github.com/Katello/katello.git/commit/d2b75839705a72a7970f732c7d7159a48125f3fe)) + * Updating the CDN URL is manifest works fine but creates some tasks which remains in planned state with success result ([#30736](https://projects.theforeman.org/issues/30736), [d62fe7d2](https://github.com/Katello/katello.git/commit/d62fe7d219c482ebfa8837debc6a5e20bbfeaa78)) + * Update ansible recommended repo version ([#30672](https://projects.theforeman.org/issues/30672), [2475b102](https://github.com/Katello/katello.git/commit/2475b102ca03bd4b83ba53e6c5962168d2c447d7)) + * Turn optimize back on for Pulp 3 yum repo syncs ([#30646](https://projects.theforeman.org/issues/30646), [a385d2f2](https://github.com/Katello/katello.git/commit/a385d2f20356f5e1db3e26a49b0fbbc714fc26ff)) + * 'The sha256 checksum did not match.' error when uploading larger files to Pulp 3 file repo through the UI ([#30462](https://projects.theforeman.org/issues/30462), [b43f2f08](https://github.com/Katello/katello.git/commit/b43f2f08b38547753f520f21340cc0b9098fe6e2)) + * Cannot see why redhat repo scanning failed ([#29933](https://projects.theforeman.org/issues/29933), [394c8b6a](https://github.com/Katello/katello.git/commit/394c8b6a3daaf59bb8942b0f555f030a4a107a52)) + +### Hosts + * Add /owners/:id/system_purpose to RHSM Proxies Controller ([#30749](https://projects.theforeman.org/issues/30749), [4173dec5](https://github.com/Katello/katello.git/commit/4173dec504c347b4a56af04b5e2ff461ad6622e6)) + * Unable to change host location ([#30702](https://projects.theforeman.org/issues/30702), [fae2cdd3](https://github.com/Katello/katello.git/commit/fae2cdd35512480ab6e55587ca48a87912cfd993)) + +### API + * Wrong API endpoint path referenced for resolving host traces ([#30700](https://projects.theforeman.org/issues/30700), [9ec9438c](https://github.com/Katello/katello.git/commit/9ec9438ce987c8841c3dcec20a908a1751e43c26)) + * API PING request return "status": "ok" even if there is a service in state failed ([#30498](https://projects.theforeman.org/issues/30498), [2b8a2115](https://github.com/Katello/katello.git/commit/2b8a2115b2ea208a45cf6abf978dab4fa01f3582)) + +### Foreman Proxy Content + * batch smart proxy content syncing causing slower syncs ([#30654](https://projects.theforeman.org/issues/30654), [a4e854cd](https://github.com/Katello/katello.git/commit/a4e854cdcb0d982a352e50640c2138140e0c25ce)) + +### Other + * Errata status is not update after applicability regeneration (pulp3) ([#30636](https://projects.theforeman.org/issues/30636), [41231b70](https://github.com/Katello/katello.git/commit/41231b70e0ae9ccf5aebc573699fc8202791188c)) + * Content Host Applicable Packages page error: NoMethodError: undefined method `where' for # ([#29563](https://projects.theforeman.org/issues/29563), [54608e1c](https://github.com/Katello/katello.git/commit/54608e1c2208dcc6273ae3d11debec9505f1391c)) diff --git a/lib/katello/version.rb b/lib/katello/version.rb index 09a313db249..a18e9ee7aa1 100644 --- a/lib/katello/version.rb +++ b/lib/katello/version.rb @@ -1,3 +1,3 @@ module Katello - VERSION = "3.16.0".freeze + VERSION = "3.16.1".freeze end