diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000000..5b670d85a047 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: ["https://www.odoo-community.org/crowdfunding/oca-open-upgrade-v20-8", "https://dir.floss.fund/view/funding/@odoo-community.org"] diff --git a/.github/actions/check-pr-dependency/action.yml b/.github/actions/check-pr-dependency/action.yml new file mode 100644 index 000000000000..7d6e6e2ce121 --- /dev/null +++ b/.github/actions/check-pr-dependency/action.yml @@ -0,0 +1,17 @@ +name: 'Check PR dependencies' +description: 'List PR dependencies in comment, set labels Dependency OK, Blocked by dependency' +inputs: + version: + description: 'OpenUpgrade version' + required: true +branding: + icon: 'git-merge' + color: 'yellow' +runs: + using: "composite" + steps: + - name: 'Run check' + run: | + pip install -r ${{ github.action_path }}/requirements.txt + python ${{ github.action_path }}/check_dependency.py --version ${{ inputs.version }} --repo ${{ github.repository }} --github-login ${{ github.actor }} + shell: sh diff --git a/.github/actions/check-pr-dependency/check_dependency.py b/.github/actions/check-pr-dependency/check_dependency.py new file mode 100644 index 000000000000..2ff3e9d51449 --- /dev/null +++ b/.github/actions/check-pr-dependency/check_dependency.py @@ -0,0 +1,162 @@ +import argparse +import os +import re +from pathlib import Path + +from github import Auth, Github +from github.PullRequestReview import PullRequestReview + +import tools + +parser = argparse.ArgumentParser() +parser.add_argument('--version', nargs='+') +parser.add_argument('--repo', default='OCA/OpenUpgrade') +parser.add_argument('--github-login', default='ocabot') +args = parser.parse_args() + +auth = None +if os.environ.get('GITHUB_TOKEN'): + auth = Auth.Token(os.environ.get('GITHUB_TOKEN')) + +g = Github(auth=auth) +repo = g.get_repo(args.repo) + +# ####################### +# Custom Function +# ####################### +def module_ok(module_name): + return module_coverage[module_name] in ["nothing_to_do", "done"] or opened_prs.get(module_name) == pr_number + +def _extract_migration_comment(migration_message): + if not migration_message: + return "", "", "" + text_reg = ( + r"(?P(\r|\n|.)*)" + r"" + "(?P(\n|.)*)" + r"<\/dependency>" + r"(?P(\r|\n|.)*)" + ) + res = re.match(text_reg, migration_message.body) + if not res: + return "", "", "" + groups = res.groupdict() + return groups["before"], groups["current_text"], groups["after"] + + +def _get_comment_or_review(pr, content): + comments = [x for x in pr.get_issue_comments() if content in x.body] + if comments: + return comments[0] + reviews = [x for x in pr.get_reviews() if content in x.body] + return reviews and reviews[0] or False + + +# #################################### +# Main Script +# #################################### + +for current_version in map(float, args.version): + tools._logger.info(f"*************************************************") + tools._logger.info(f"Update Dependencies for Version {current_version}") + tools._logger.info(f"*************************************************") + + # Get Opened PRs + done_prs, opened_prs = tools._get_prs(repo, current_version) + + # Get Done modules + module_coverage = tools._get_module_coverage(repo, current_version) + + + for module_name, pr_number in opened_prs.items(): + tools._logger.info(f"Handle PR #{pr_number} for module '{module_name}'") + manifest = tools._get_manifest_module(repo, current_version, module_name) + if not manifest: + tools._logger.info(f"the module {module_name} disappeared in the new version.") + continue + + depends = manifest.get("depends") + pr = repo.get_pull(pr_number) + + migration_message = _get_comment_or_review(pr, "ocabot migration") + current_labels = [x.name for x in pr.labels] + new_labels = current_labels.copy() + + log_message = f"PR #{pr_number} for {module_name}:" + if all([module_ok(x) for x in depends]): + # all dependencies are OK, mark the PR as OK. + label_to_remove = "Blocked by dependency" + label_to_add = "Dependency OK" + else: + # mark the PR as blocked + label_to_remove = "Dependency OK" + label_to_add = "Blocked by dependency" + + if label_to_remove in new_labels: + log_message += f" Remove '{label_to_remove}'" + new_labels.remove(label_to_remove) + + if label_to_add not in new_labels: + log_message += f" Set '{label_to_add}'" + new_labels.append(label_to_add) + + if new_labels != current_labels: + tools._logger.info(log_message) + pr.set_labels(*new_labels) + + if not migration_message: + tools._logger.warning(f"Migration message not found in {pr.number}") + continue + + before_text, current_text, after_text = _extract_migration_comment( + migration_message + ) + + dep_modules_ok = [x for x in depends if x in done_prs] + dep_modules_pending = [x for x in depends if x in opened_prs] + dep_modules_ko = [ + x for x in depends if x not in dep_modules_ok + dep_modules_pending + ] + new_text = "" + for dep_module in dep_modules_ok: + new_text += f"\n- {dep_module} : #{done_prs[dep_module]}" + for dep_module in dep_modules_pending: + new_text += f"\n- {dep_module} : #{opened_prs[dep_module]}" + for dep_module in dep_modules_ko: + new_text += f"\n- {dep_module} : TODO" + if module_ok(dep_module): + tools._logger.error(f"{dep_module} not found in the issue 'Migration to version {current_version}'") + + + if new_text: + new_text = "Depends on : \n" + new_text + + new_body = "" + if not current_text and new_text: + tools._logger.info( + f"PR#{pr.number} - {pr.title}: ADDING Text in migration message." + ) + if migration_message.user.login != args.github_login: + tools._logger.warning( + f"The message has been written by @{migration_message.user.login}" + ) + new_body = ( + migration_message.body + "\n\n" + new_text + "" + ) + elif new_text != current_text: + tools._logger.info( + f"PR#{pr.number} - {pr.title}: Replacing text in migration message." + ) + if migration_message.user.login != args.github_login: + tools._logger.warning( + f"The message has been written by @{migration_message.user.login}" + ) + new_body = ( + before_text + "" + new_text + "" + after_text + ) + + if new_body: + if type(migration_message) is PullRequestReview: + tools._logger.warning("Unable to edit the PR due to PyGithub limitation.") + continue + migration_message.edit(new_body) diff --git a/.github/actions/check-pr-dependency/requirements.txt b/.github/actions/check-pr-dependency/requirements.txt new file mode 100644 index 000000000000..945b116ad3c5 --- /dev/null +++ b/.github/actions/check-pr-dependency/requirements.txt @@ -0,0 +1 @@ +PyGithub diff --git a/.github/actions/check-pr-dependency/tools.py b/.github/actions/check-pr-dependency/tools.py new file mode 100644 index 000000000000..27aee3152866 --- /dev/null +++ b/.github/actions/check-pr-dependency/tools.py @@ -0,0 +1,90 @@ +import logging +import re +import sys +import urllib + +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(stream=sys.stdout)], +) + +_logger = logging.getLogger() + + +def _get_prs(repo, version): + opened_prs = {} + done_prs = {} + + line_reg = ( + r"- \[(?P |x)\] (\(del\) )?(\(new\) )?" + r"(?P\w+).*(pull\/|#)(?P\d+)" + ) + issue_title = f"Migration to version {version:.1f}" + + _logger.info(f"Get PRs referenced in '{issue_title}' ...") + # Get main issue to have PRs numbers + for issue in repo.get_issues(): + if issue.title == issue_title: + for line in issue.body.split("\n"): + if not line.startswith("- ["): + continue + res = re.match(line_reg, line) + if not res: + _logger.debug(f"Unable to parse correctly '{line}'") + continue + groups = res.groupdict() + pr_number = int(groups["pr_number"]) + if groups["done"] == "x": + done_prs[groups["module_name"]] = pr_number + else: + opened_prs[groups["module_name"]] = pr_number + _logger.info( + f">> found {len(done_prs)} done PRs" f" and {len(opened_prs)} Opened PRs" + ) + return done_prs, opened_prs + + +def _get_module_coverage(repo, version): + file_name = f"modules{(version - 1) * 10:.0f}-{(version) * 10:.0f}.rst" + url = ( + f"https://raw.githubusercontent.com/OCA/OpenUpgrade" + f"/{version:.1f}/docsource/" + f"{file_name}" + ) + module_coverage = {} + _logger.info(f"Get and parse {file_name} to know coverage state ...") + page = urllib.request.urlopen(url).read().decode() + for line in page.split("\n"): + line = line.replace("|del|", "").replace("|new|", "") + if not line.startswith("|"): + continue + res = line.split("|") + module_name = res[1].strip() + status = res[2].strip() + if module_name == "Module": + continue + if status == "": + module_coverage[module_name] = "to_do" + elif status == "Nothing to do": + module_coverage[module_name] = "nothing_to_do" + elif status.startswith("Done"): + module_coverage[module_name] = "done" + else: + continue + # TODO:fix me + raise Exception("Status '%s' has not been analyzed." % (status)) + _logger.info(f">> found {len(module_coverage)} modules.") + return module_coverage + + +def _get_manifest_module(repo, version, module_name): + url = ( + f"https://raw.githubusercontent.com/odoo/" + f"odoo/{version:.1f}/addons/{module_name}/__manifest__.py" + ) + try: + page = urllib.request.urlopen(url).read().decode() + except urllib.error.HTTPError: + return + return eval(page) diff --git a/.github/workflows/check_pr_dependency.yml b/.github/workflows/check_pr_dependency.yml new file mode 100644 index 000000000000..37c3993d367f --- /dev/null +++ b/.github/workflows/check_pr_dependency.yml @@ -0,0 +1,19 @@ +name: Check PR dependencies + +on: + issues: + types: [edited] + +jobs: + check_dependency: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + if: ${{ github.event.issue.state == 'open' && startsWith( github.event.issue.title, 'Migration to version ' ) }} + - name: Check dependency + if: ${{ github.event.issue.state == 'open' && startsWith( github.event.issue.title, 'Migration to version ' ) }} + uses: ./.github/actions/check-pr-dependency + with: + version: ${{ github.event.issue.milestone.title }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/documentation-commit.yml b/.github/workflows/documentation-commit.yml new file mode 100644 index 000000000000..b269f9611796 --- /dev/null +++ b/.github/workflows/documentation-commit.yml @@ -0,0 +1,69 @@ +# On each push in 20.0 branch, +# AND if the coverage file changed, +# build documentation branch and commit the changes +# so that the changes are visible on the website +# https://oca.github.io/OpenUpgrade/ + +name: Build and commit documentation + +on: + push: + paths: ["docsource/modules190-200.rst"] + +jobs: + documentation-commit: + runs-on: ubuntu-latest + steps: + - name: Check out OpenUpgrade Documentation + uses: actions/checkout@v2 + with: + ref: "documentation" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Check out Odoo + uses: actions/checkout@v2 + with: + repository: odoo/odoo + ref: "20.0" + fetch-depth: 1 + path: odoo + - name: Configuration + run: | + sudo apt update + sudo apt install \ + expect \ + expect-dev \ + libevent-dev \ + libldap2-dev \ + libsasl2-dev \ + libxml2-dev \ + libxslt1-dev \ + nodejs \ + python3-lxml \ + python3-passlib \ + python3-psycopg2 \ + python3-serial \ + python3-simplejson \ + python3-werkzeug \ + python3-yaml \ + unixodbc-dev + - name: Requirements Installation + run: | + export PIP_CONSTRAINT=pip-constraint.txt + echo 'setuptools < 82.0.0' > pip-constraint.txt + sed -i -E "s/(gevent==)21\.8\.0( ; sys_platform != 'win32' and python_version == '3.10')/\122.10.2\2/;s/(greenlet==)1.1.2( ; sys_platform != 'win32' and python_version == '3.10')/\12.0.2\2/" odoo/requirements.txt + pip install -q -r odoo/requirements.txt + pip install -r ./requirements.txt + - name: OpenUpgrade Docs + run: | + # try to build the documentation + sh ./build_openupgrade_docs + - name: Commit changes + uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 #v9 + with: + add: "docs" + default_author: github_actions + message: "[UPD] HTML documentation" diff --git a/.github/workflows/generate-analysis-cron.yml b/.github/workflows/generate-analysis-cron.yml new file mode 100644 index 000000000000..cd073cc22f26 --- /dev/null +++ b/.github/workflows/generate-analysis-cron.yml @@ -0,0 +1,40 @@ +name: Generate analysis files - cron and push + +on: + push: + # when module merges are added to apriori, this + # might cause changes in the analysis + paths: ["openupgrade_scripts/apriori.py"] + schedule: + # every monday at 04:00 + - cron: "0 4 * * 1" + workflow_dispatch: + inputs: + version: + description: 'Version' + required: true + type: choice + options: + - all + - 18.0 + - 19.0 + +jobs: + generate-analysis18: + name: Generate analysis 18.0 + if: | + github.repository == 'OCA/OpenUpgrade' && (github.event_name == 'schedule' || github.ref_name == '18.0') || + github.event_name == 'workflow_dispatch' && (inputs.version == '18.0' || inputs.version == 'all') + uses: ./.github/workflows/generate-analysis.yml + with: + from_version: "17.0" + to_version: "18.0" + generate-analysis19: + name: Generate analysis 19.0 + if: | + github.repository == 'OCA/OpenUpgrade' && (github.event_name == 'schedule' || github.ref_name == '19.0') || + github.event_name == 'workflow_dispatch' && (inputs.version == '19.0' || inputs.version == 'all') + uses: ./.github/workflows/generate-analysis.yml + with: + from_version: "18.0" + to_version: "19.0" diff --git a/.github/workflows/generate-analysis.yml b/.github/workflows/generate-analysis.yml new file mode 100644 index 000000000000..8ec06c1c7cce --- /dev/null +++ b/.github/workflows/generate-analysis.yml @@ -0,0 +1,236 @@ +name: Generate analysis files + +on: + workflow_call: + inputs: + from_version: + required: true + type: string + from_version_requirements: + default: 'gevent==22.10.2 greenlet==2.0.2' + type: string + from_version_requirements_sed: + default: '/^gevent\>/d; /^greenlet\>/d' + type: string + to_version: + required: true + type: string + to_version_requirements: + default: 'gevent==22.10.2 greenlet==2.0.2' + type: string + to_version_requirements_sed: + default: '/^gevent\>/d; /^greenlet\>/d' + type: string + python_version: + default: "3.10" + type: string + postgres_version: + default: "14" + type: string + +jobs: + generate-analysis: + name: Generate analysis ${{ inputs.to_version }} + runs-on: ubuntu-latest + env: + PGHOST: "localhost" + PGPASSWORD: "odoo" + PGUSER: "odoo" + FROM_VERSION: "${{ inputs.from_version }}" + TO_VERSION: "${{ inputs.to_version }}" + services: + postgres: + image: postgres:${{ inputs.postgres_version }} + env: + POSTGRES_USER: odoo + POSTGRES_PASSWORD: odoo + ports: + - 5432:5432 + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "${{ inputs.python_version }}" + - name: Install required packages + run: | + sudo apt update + sudo apt install \ + expect \ + expect-dev \ + libevent-dev \ + libldap2-dev \ + libsasl2-dev \ + libxml2-dev \ + libxslt1-dev \ + nodejs \ + python3-lxml \ + python3-passlib \ + python3-psycopg2 \ + python3-serial \ + python3-simplejson \ + python3-werkzeug \ + python3-yaml \ + unixodbc-dev + - name: Checkout previous OpenUpgrade + uses: actions/checkout@v4 + with: + ref: "${{ env.FROM_VERSION }}" + path: openupgrade-${{ env.FROM_VERSION }} + - name: Checkout OpenUpgrade from current branch + if: github.event_name != 'schedule' && startsWith(github.ref_name, env.TO_VERSION) + uses: actions/checkout@v4 + with: + path: openupgrade-${{ env.TO_VERSION }} + - name: Checkout current OpenUpgrade + if: github.event_name == 'schedule' || !startsWith(github.ref_name, env.TO_VERSION) + uses: actions/checkout@v4 + with: + ref: "${{ env.TO_VERSION }}" + path: openupgrade-${{ env.TO_VERSION }} + - name: Install previous Odoo + run: | + wget -q -O- https://github.com/oca/ocb/archive/refs/heads/$FROM_VERSION.tar.gz | tar -xz + wget -q -O- https://github.com/oca/server-tools/archive/refs/heads/$FROM_VERSION.tar.gz | tar -xz + python -mvenv env-$FROM_VERSION + . env-$FROM_VERSION/bin/activate + export PIP_CONSTRAINT=pip-constraint.txt + echo 'setuptools < 82.0.0' > pip-constraint.txt + pip install ${{ inputs.from_version_requirements }} + sed -iE '${{ inputs.from_version_requirements_sed }}' OCB-$FROM_VERSION/requirements.txt + pip install -r OCB-$FROM_VERSION/requirements.txt + pip install ./OCB-$FROM_VERSION + pip install -r server-tools-$FROM_VERSION/requirements.txt + pip install -r openupgrade-$FROM_VERSION/requirements.txt + # this is for l10n_eg_edi_eta which crashes without it + pip install asn1crypto + # this is for cloud_storage_google + pip install "google-auth<2.48.0" + odoo -s -c odoo-$FROM_VERSION.cfg --addons-path server-tools-$FROM_VERSION,openupgrade-$FROM_VERSION --stop-after-init + - name: Install current Odoo + run: | + wget -q -O- https://github.com/oca/ocb/archive/refs/heads/$TO_VERSION.tar.gz | tar -xz + wget -q -O- https://github.com/oca/server-tools/archive/refs/heads/$TO_VERSION.tar.gz | tar -xz + python -mvenv env-$TO_VERSION + . env-$TO_VERSION/bin/activate + export PIP_CONSTRAINT=pip-constraint.txt + echo 'setuptools < 82.0.0' > pip-constraint.txt + pip install ${{ inputs.to_version_requirements }} + sed -iE '${{ inputs.to_version_requirements_sed }}' OCB-$TO_VERSION/requirements.txt + pip install -r OCB-$TO_VERSION/requirements.txt + pip install ./OCB-$TO_VERSION + pip install -r server-tools-$TO_VERSION/requirements.txt + pip install -r openupgrade-$TO_VERSION/requirements.txt + # this is for l10n_eg_edi_eta which crashes without it + pip install asn1crypto + # this is for cloud_storage_google + pip install "google-auth<2.48.0" + odoo -s -c odoo-$TO_VERSION.cfg --addons-path server-tools-$TO_VERSION,openupgrade-$TO_VERSION --stop-after-init + - name: Create previous Odoo database + env: + ODOO: env-${{ env.FROM_VERSION }}/bin/odoo -c odoo-${{ env.FROM_VERSION }}.cfg -d ${{ env.FROM_VERSION }} + ODOO_SHELL: env-${{ env.FROM_VERSION }}/bin/odoo shell -c odoo-${{ env.FROM_VERSION }}.cfg -d ${{ env.FROM_VERSION }} + run: | + $ODOO --without-demo=all -i upgrade_analysis --stop-after-init + $ODOO_SHELL < "$PR_BODY" + + for module in $( + git -C $OPENUPGRADE_DIR status --short | awk '{print $2}' |\ + grep --invert-match upgrade_general_log.txt |\ + awk -F/ '{print $3}' | sort -u + ); do + if grep -qE "\<$module\>.*\<((Done)|(No))" $OPENUPGRADE_DIR/docsource/modules*; then + echo Done module $module was updated + echo "- $module" >> $PR_BODY + else + echo $module is not done yet + fi + done + + if ! grep -q - $PR_BODY; then + echo None >> $PR_BODY + fi + + echo "body-file=$PR_BODY" >> "$GITHUB_OUTPUT" + - name: Get number of milestone + id: get_milestone + run: | + echo "number=$( + wget -qO- https://api.github.com/repos/${{ github.repository }}/milestones|jq '.[] | select(.title=="${{ env.TO_VERSION }}") .number' + )" >> "$GITHUB_OUTPUT" + - name: Create PR + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e #v7 + with: + base: "${{ env.TO_VERSION }}" + body-path: "${{ steps.generate_body.outputs.body-file }}" + branch: "${{ env.TO_VERSION }}-update-analysis-bot" + commit-message: "[IMP] Update analysis files" + delete-branch: true + milestone: "${{ steps.get_milestone.outputs.number }}" + path: "openupgrade-${{ env.TO_VERSION }}" + title: "[${{ env.TO_VERSION }}][IMP] Update analysis files" diff --git a/.github/workflows/generate-testdb-cron.yml b/.github/workflows/generate-testdb-cron.yml new file mode 100644 index 000000000000..be49e54d5d35 --- /dev/null +++ b/.github/workflows/generate-testdb-cron.yml @@ -0,0 +1,40 @@ +name: Generate test databases - cron and manually + +on: + workflow_dispatch: + inputs: + version: + description: 'Version' + required: true + type: choice + options: + - all + - 16.0 + - 17.0 + - 18.0 + schedule: + # every first of month + - cron: "0 4 1 * *" + +jobs: + generate-testdb18: + name: Generate test database 18.0 + if: (github.repository == 'OCA/OpenUpgrade' && github.event_name == 'schedule') || inputs.version == '18.0' || inputs.version == 'all' + uses: ./.github/workflows/generate-testdb.yml + with: + version: "18.0" + exclude_modules: "['payment_alipay', 'payment_ogone', 'payment_payulatam', 'payment_payumoney']" + generate-testdb17: + name: Generate test database 17.0 + if: (github.repository == 'OCA/OpenUpgrade' && github.event_name == 'schedule') || inputs.version == '17.0' || inputs.version == 'all' + uses: ./.github/workflows/generate-testdb.yml + with: + version: "17.0" + exclude_modules: "['payment_alipay', 'payment_ogone', 'payment_payulatam', 'payment_payumoney']" + generate-testdb16: + name: Generate test database 16.0 + if: (github.repository == 'OCA/OpenUpgrade' && github.event_name == 'schedule') || inputs.version == '16.0' || inputs.version == 'all' + uses: ./.github/workflows/generate-testdb.yml + with: + version: "16.0" + exclude_modules: "['payment_alipay', 'payment_ogone', 'payment_payulatam', 'payment_payumoney']" diff --git a/.github/workflows/generate-testdb.yml b/.github/workflows/generate-testdb.yml new file mode 100644 index 000000000000..15ef8bd1eb3a --- /dev/null +++ b/.github/workflows/generate-testdb.yml @@ -0,0 +1,76 @@ +name: Generate test database + +on: + workflow_call: + inputs: + version: + required: true + type: string + python_version: + default: "3.10" + type: string + postgres_version: + default: "14" + type: string + exclude_modules: + default: "[]" + type: string + release_tag: + default: "databases" + type: string +jobs: + generate-testdb: + name: Generate test database ${{ inputs.version }} + runs-on: ubuntu-22.04 + container: ghcr.io/oca/oca-ci/py${{ inputs.python_version }}-ocb${{ inputs.version }}:latest + env: + PGHOST: "postgres" + PGPASSWORD: "odoo" + PGUSER: "odoo" + VERSION: "${{ inputs.version }}" + DBNAME: "openupgrade" + services: + postgres: + image: postgres:${{ inputs.postgres_version }} + env: + POSTGRES_USER: odoo + POSTGRES_PASSWORD: odoo + ports: + - 5432:5432 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install extra dependencies + run: | + # this is for l10n_eg_edi_eta which crashes without it + pip install asn1crypto + # this is for cloud_storage_google + pip install "google-auth<2.48.0" + # this is for account_peppol + pip install phonenumbers + # for pushing release + apt install gh -y + - name: Install all modules and an extra language + run: | + odoo -d $DBNAME -i base --stop-after-init + odoo shell -d $DBNAME < /dev/null)"; then + for snippet in openupgrade/openupgrade_scripts/scripts/*/tests/data*.py; do + odoo-old/odoo-bin shell -d $DB < $snippet + done + fi + - name: OpenUpgrade test + id: run_migration + run: | + # select modules and perform the upgrade + MODULES_OLD=$(\ + sed -n '/^+========/,$p' \ + openupgrade/docsource/modules190-200.rst \ + | grep "Done\|Partial\|Nothing" \ + | grep -v "theme_" \ + | sed -rn 's/((^\| *\|del\| *)|^\| *)([0-9a-z_]*)[ \|].*/\3/g p' \ + | sed '/^\s*$/d' \ + | paste -d, -s) + MODULES_NEW=$(\ + sed -n '/^+========/,$p' \ + openupgrade/docsource/modules190-200.rst \ + | grep "Done\|Partial\|Nothing" \ + | grep -v "theme_" \ + | sed -rn 's/((^\| *\|new\| *)|^\| *)([0-9a-z_]*)[ \|].*/\3/g p' \ + | sed '/^\s*$/d' \ + | paste -d, -s) + echo "modules_old=$MODULES_OLD" >> $GITHUB_OUTPUT + echo "modules_new=$MODULES_NEW" >> $GITHUB_OUTPUT + if [ -z "$MODULES_NEW" ]; then + echo "No modules to test yet" + exit + fi + REQUEST="update ir_module_module set state='uninstalled' \ + where name not in ('$(echo $MODULES_OLD | sed -e "s/,/','/g")')" + echo Set the modules as not installable if they are not in the following list : $MODULES_OLD + echo Running $REQUEST + psql $DB -c "$REQUEST" + ADDONS_PATHS="\ + $GITHUB_WORKSPACE/odoo/addons \ + $GITHUB_WORKSPACE/odoo/odoo/addons \ + $GITHUB_WORKSPACE/openupgrade" + echo Execution of Openupgrade with the update of the following modules : $MODULES_NEW + # Silence redundant logs from unlinking records (1 line is enough) + # to prevent log overflow + coverage run \ + --source=$GITHUB_WORKSPACE/openupgrade/openupgrade_scripts/scripts \ + $ODOO \ + --addons-path=`echo $ADDONS_PATHS | awk -v OFS="," '$1=$1'` \ + --database=$DB \ + --db_host=$DB_HOST \ + --db_password=$DB_PASSWORD \ + --db_port=$DB_PORT \ + --db_user=$DB_USERNAME \ + --load=base,web,openupgrade_framework \ + --test-enable \ + --test-tags openupgrade \ + --log-handler odoo.models.unlink:WARNING \ + --stop-after-init \ + --without-demo=$MODULES_NEW \ + --update=$MODULES_NEW + - name: Generate coverage.xml + if: ${{ steps.run_migration.outputs.modules_new != '' }} + run: coverage xml + - uses: codecov/codecov-action@v4 + if: ${{ steps.run_migration.outputs.modules_new != '' }} + with: + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000000..22a8861c737d --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +openupgrade_scripts/scripts/*/*/noupdate_changes*.xml +docsource/*.rst +docs diff --git a/openupgrade_framework/README.rst b/openupgrade_framework/README.rst new file mode 100644 index 000000000000..e26c73a2ca96 --- /dev/null +++ b/openupgrade_framework/README.rst @@ -0,0 +1,214 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +===================== +Openupgrade Framework +===================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:5798eba54677737dfb52eec94d10de67ba298ebfdda05d8173dc4a802c8be38e + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2FOpenUpgrade-lightgray.png?logo=github + :target: https://github.com/OCA/OpenUpgrade/tree/20.0/openupgrade_framework + :alt: OCA/OpenUpgrade +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/OpenUpgrade-20-0/OpenUpgrade-20-0-openupgrade_framework + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/OpenUpgrade&target_branch=20.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module is a technical module that contains a number of +monkeypatches to improve the behaviour of Odoo when migrating your +database using the OpenUpgrade migration scripts: + +- Prevent dropping columns or tables in the database when fields or + models are obsoleted in the Odoo data model of the target release. + After the migration, you can review and delete unused database tables + and columns using database_cleanup. See + https://odoo-community.org/shop/product/database-cleanup-918 +- When data records are deleted during the migration (such as views or + other system records), this is done in a secure mode. If the deletion + fails because of some unforeseen dependency, the deletion will be + cancelled and a message is logged, after which the migration + continues. +- Prevent a number of log messages that do not apply when using + OpenUpgrade. +- Suppress log messages about failed view validation, which are to be + expected during a migration. +- Run migration scripts for modules that are installed as new + dependencies of upgraded modules (when there are such scripts for + those particular modules) +- Production databases generated with demo data, will be transformed to + non-demo ones. If you want to avoid that, you have to pass through the + environment variable OPENUPGRADE_USE_DEMO, the value "yes". + +For detailed documentation see: + +- https://github.com/OCA/OpenUpgrade/ +- https://oca.github.io/OpenUpgrade + +**Table of contents** + +.. contents:: + :local: + +Installation +============ + +This module does not need to be installed on a database. It simply needs +to be available via your ``addons-path``. + +Configuration +============= + +- call your odoo instance with the option + ``--load=base,web,openupgrade_framework`` + +or + +- add the key to your configuration file: + +.. code:: shell + + [options] + server_wide_modules = web,openupgrade_framework + +When you load the module in either way of these ways, and you have the +openupgrade_scripts module in your addons path available, the +--upgrade-path option of Odoo will be set automatically to the location +of the OpenUpgrade migration scripts. + +Development +=========== + +The odoo_patch folder contains python files in a tree that mimicks the +folder tree of the Odoo project. It contains a number of monkey patches +to improve the migration of an Odoo database between two major versions. + +So far, we are able to make everything work without overwriting large +blocks of code, but if larger patches need to be added, please use the +method described below: + +To see the patches added, you can use ``meld`` tools: + +``meld PATH_TO_ODOO_FOLDER/odoo/ PATH_TO_OPENUPGRADE_FRAMEWORK_MODULE/odoo_patch`` + +To make more easy the diff analysis : + +- Make sure the python files has the same path as the original one. +- Keep the same indentation as the original file. (using ``if True:`` if + required) +- Add the following two lines at the beginning of your file, to avoid + flake8 / pylint errors + +.. code:: python + + # flake8: noqa + # pylint: skip-file + +- When you want to change the code. add the following tags: + +For an addition: + +.. code:: python + + # + some code... + # + +For a change: + +.. code:: python + + # + some code... + # + +For a removal: + +.. code:: python + + # + # Comment the code, instead of removing it. + # + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Therp BV +* Opener B.V. +* GRAP +* Hunki Enterprises BV + +Contributors +------------ + +- Stefan Rijnhart +- Sylvain LE GAL + +Other credits +------------- + +Many developers have contributed to the OpenUpgrade framework in its +previous incarnation. Their original contributions may no longer needed, +or they are no longer recognizable in their current form but OpenUpgrade +would not have existed at this point without them. + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-legalsylvain| image:: https://github.com/legalsylvain.png?size=40px + :target: https://github.com/legalsylvain + :alt: legalsylvain +.. |maintainer-StefanRijnhart| image:: https://github.com/StefanRijnhart.png?size=40px + :target: https://github.com/StefanRijnhart + :alt: StefanRijnhart +.. |maintainer-hbrunn| image:: https://github.com/hbrunn.png?size=40px + :target: https://github.com/hbrunn + :alt: hbrunn + +Current `maintainers `__: + +|maintainer-legalsylvain| |maintainer-StefanRijnhart| |maintainer-hbrunn| + +This module is part of the `OCA/OpenUpgrade `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/openupgrade_framework/__init__.py b/openupgrade_framework/__init__.py new file mode 100644 index 000000000000..97a704fd9e63 --- /dev/null +++ b/openupgrade_framework/__init__.py @@ -0,0 +1,33 @@ +import logging +import os + +from odoo.modules import get_module_path +from odoo.tools import config + +from . import odoo_patch + +if not config.get("upgrade_path"): + path = get_module_path("openupgrade_scripts", display_warning=False) + if path and os.path.isdir(os.path.join(path, "scripts")): + logging.getLogger(__name__).info( + "Setting upgrade_path to the scripts directory inside the module " + "location of openupgrade_scripts" + ) + config["upgrade_path"] = os.path.join(path, "scripts") + + +def openupgrade_test(cls): + """ + Set attributes on a test class necessary for the test framework + Use as decorator on test classes in openupgrade_scripts/scripts/*/tests/test_*.py + """ + tags = getattr(cls, "test_tags", None) or set() + if "openupgrade" not in tags: + tags.add("openupgrade") + if not any(t.endswith("_install") for t in tags): + tags.add("at_install") + cls.test_tags = tags + cls.test_module = cls.__module__.split(".")[2] + cls.test_class = cls.__name__ + cls.test_sequence = 0 + return cls diff --git a/openupgrade_framework/__manifest__.py b/openupgrade_framework/__manifest__.py new file mode 100644 index 000000000000..09f4bf7a264a --- /dev/null +++ b/openupgrade_framework/__manifest__.py @@ -0,0 +1,18 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "Openupgrade Framework", + "summary": "Module to add in the server_wide_modules " + "option to make upgrades between two major revisions.", + "author": "Odoo Community Association (OCA), Therp BV, Opener B.V., GRAP, " + "Hunki Enterprises BV", + "maintainers": ["legalsylvain", "StefanRijnhart", "hbrunn"], + "website": "https://github.com/OCA/OpenUpgrade", + "category": "Migration", + "version": "20.0.1.0.0", + "license": "AGPL-3", + "depends": ["base"], + "images": ["static/description/banner.jpg"], + "external_dependencies": {"python": ["openupgradelib"]}, + "installable": True, +} diff --git a/openupgrade_framework/odoo_patch/__init__.py b/openupgrade_framework/odoo_patch/__init__.py new file mode 100644 index 000000000000..3c691cd11703 --- /dev/null +++ b/openupgrade_framework/odoo_patch/__init__.py @@ -0,0 +1 @@ +from . import odoo diff --git a/openupgrade_framework/odoo_patch/odoo/__init__.py b/openupgrade_framework/odoo_patch/odoo/__init__.py new file mode 100644 index 000000000000..18ff659459d2 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/__init__.py @@ -0,0 +1 @@ +from . import addons, modules, orm diff --git a/openupgrade_framework/odoo_patch/odoo/addons/__init__.py b/openupgrade_framework/odoo_patch/odoo/addons/__init__.py new file mode 100644 index 000000000000..0e44449338cf --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/__init__.py @@ -0,0 +1 @@ +from . import base diff --git a/openupgrade_framework/odoo_patch/odoo/addons/base/__init__.py b/openupgrade_framework/odoo_patch/odoo/addons/base/__init__.py new file mode 100644 index 000000000000..0650744f6bc6 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/base/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/openupgrade_framework/odoo_patch/odoo/addons/base/models/__init__.py b/openupgrade_framework/odoo_patch/odoo/addons/base/models/__init__.py new file mode 100644 index 000000000000..5ec7e0740f90 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/base/models/__init__.py @@ -0,0 +1,3 @@ +from . import ir_model +from . import ir_module +from . import ir_ui_view diff --git a/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_model.py b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_model.py new file mode 100644 index 000000000000..99b3487e939a --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_model.py @@ -0,0 +1,58 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from openupgradelib import openupgrade + +from odoo import models + +from odoo.addons.base.models.ir_model import ( + IrModel, + IrModelFields, + IrModelRelation, +) + + +def _drop_table(self): + """Never drop tables""" + for model in self: + if self.env.get(model.model) is not None: + openupgrade.message( + self.env.cr, + self.env.context.get("module", "Unknown"), + False, + False, + "Not dropping the table or view of model %s", + model.model, + ) + + +IrModel._drop_table = _drop_table + + +def _drop_column(self): + """Never drop columns""" + for field in self: + if field.name in models.MAGIC_COLUMNS: + continue + openupgrade.message( + self.env.cr, + self.env.context.get("module", "Unknown"), + False, + False, + "Not dropping the column of field %s of model %s", + field.name, + field.model, + ) + continue + + +IrModelFields._drop_column = _drop_column + + +def _module_data_uninstall(self): + """Don't delete many2many relation tables. Only unlink the + ir.model.relation record itself. + """ + self.unlink() + + +IrModelRelation._module_data_uninstall = _module_data_uninstall diff --git a/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_module.py b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_module.py new file mode 100644 index 000000000000..b71e0f78534f --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_module.py @@ -0,0 +1,50 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from odoo import api + +from odoo.addons.base.models.ir_module import IrModuleModule + + +@api.model +def update_list(self): + """ + Mark auto_install modules as to install if all their dependencies are some kind of + installed. + Ignore localization modules that are set to auto_install + """ + result = IrModuleModule.update_list._original_method(self) + new_auto_install_modules = self.browse([]) + for module in self.env["ir.module.module"].search( + [ + ("auto_install", "=", True), + ("state", "=", "uninstalled"), + ("name", "not like", ("l10n_%")), + ] + ): + if all( + state in ("to upgrade", "to install", "installed") + for state in module.dependencies_id.mapped("state") + ): + new_auto_install_modules |= module + if new_auto_install_modules: + new_auto_install_modules.button_install() + return result + + +def check_external_dependencies(self, module_name, newstate="to install"): + try: + IrModuleModule.check_external_dependencies._original_method( + self, module_name, newstate=newstate + ) + except AttributeError: # pylint: disable=except-pass + # this happens when a module is installed that doesn't exist in the new version + pass + + +update_list._original_method = IrModuleModule.update_list +IrModuleModule.update_list = update_list +check_external_dependencies._original_method = ( + IrModuleModule.check_external_dependencies +) +IrModuleModule.check_external_dependencies = check_external_dependencies diff --git a/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_ui_view.py b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_ui_view.py new file mode 100644 index 000000000000..8b82f5f3a84a --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/addons/base/models/ir_ui_view.py @@ -0,0 +1,94 @@ +# Copyright 2024 Viindoo Technology Joint Stock Company (Viindoo) +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import logging + +from odoo.exceptions import ValidationError +from odoo.modules.module import get_resource_from_path +from odoo.tools import mute_logger + +from odoo.addons.base.models.ir_ui_view import IrUiView + +_logger = logging.getLogger(__name__) + + +def _check_xml(self): + """Don't raise or log exceptions in view validation unless explicitly + requested. Mute warnings about views which are common during migration.""" + with mute_logger("odoo.addons.base.models.ir_ui_view"): + try: + return IrUiView._check_xml._original_method(self) + except ValidationError as e: + _logger.warning( + "Can't render custom view %s for model %s. " + "Assuming you are migrating between major versions of Odoo. " + "Please review the view contents manually after the migration.\n" + "Error: %s", + self.xml_id, + self.model, + e, + ) + + +def _raise_view_error( + self, message, node=None, *, from_exception=None, from_traceback=None +): + """Don't raise or log exceptions in view validation unless explicitly + requested + """ + raise_exception = self.env.context.get("raise_view_error") + to_mute = "odoo.addons.base.models.ir_ui_view" if raise_exception else "not_muted" + with mute_logger(to_mute): + try: + return IrUiView._raise_view_error._original_method( + self, + message, + node=node, + from_exception=from_exception, + from_traceback=from_traceback, + ) + except ValueError as e: + _logger.warning( + "Can't render custom view %s for model %s. " + "Assuming you are migrating between major versions of Odoo. " + "Please review the view contents manually after the migration.\n" + "Error: %s", + self.xml_id, + self.model, + e, + ) + + +def _check_field_paths(self, node, field_paths, model_name, use): + """Ignore UnboundLocalError when we squelched the raise about missing fields""" + try: + return IrUiView._check_field_paths._original_method( + self, node, field_paths, model_name, use + ) + except UnboundLocalError: # pylint: disable=except-pass + pass + + +def _inverse_arch(self): + """ + Remove install_filename from context if it's from openupgrade_scripts. + Without this, arch_fs will point to openupgrade_scripts' file which most likely + won't exist when the migrated database is deployed, which breaks resetting views + """ + if "install_filename" in self.env.context: + path_info = get_resource_from_path(self.env.context["install_filename"]) + if path_info and path_info[0] == "openupgrade_scripts": + self = self.with_context( # pylint: disable=context-overridden + {k: v for k, v in self.env.context.items() if k != "install_filename"} + ) + return _inverse_arch._original_method(self) + + +_check_xml._original_method = IrUiView._check_xml +IrUiView._check_xml = _check_xml +_raise_view_error._original_method = IrUiView._raise_view_error +IrUiView._raise_view_error = _raise_view_error +_check_field_paths._original_method = IrUiView._check_field_paths +IrUiView._check_field_paths = _check_field_paths +_inverse_arch._original_method = IrUiView._inverse_arch +IrUiView._inverse_arch = _inverse_arch diff --git a/openupgrade_framework/odoo_patch/odoo/modules/__init__.py b/openupgrade_framework/odoo_patch/odoo/modules/__init__.py new file mode 100644 index 000000000000..e060fc2ed96b --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/__init__.py @@ -0,0 +1 @@ +from . import module_graph, loading, migration diff --git a/openupgrade_framework/odoo_patch/odoo/modules/loading.py b/openupgrade_framework/odoo_patch/odoo/modules/loading.py new file mode 100644 index 000000000000..3a5424f476ce --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/loading.py @@ -0,0 +1,18 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import odoo.modules.loading + + +def load_module_graph(env, graph, *args, **kwargs): + """ + Force run pre-migration scripts for modules being installed + """ + env.registry._force_upgrade_scripts.update({package.name for package in graph}) + return odoo.modules.loading.load_module_graph._original_method( + env, graph, *args, **kwargs + ) + + +load_module_graph._original_method = odoo.modules.loading.load_module_graph +odoo.modules.loading.load_module_graph = load_module_graph diff --git a/openupgrade_framework/odoo_patch/odoo/modules/migration.py b/openupgrade_framework/odoo_patch/odoo/modules/migration.py new file mode 100644 index 000000000000..fdc6f09eda1b --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/migration.py @@ -0,0 +1,16 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +from odoo.modules.migration import MigrationManager + + +def _get_files(self): + """Turns out Odoo SA sometimes add migration scripts that interfere with + OpenUpgrade. Those we filter out here""" + MigrationManager._get_files._original_method(self) + to_exclude = [("analytic", "1.2")] + for addon, version in to_exclude: + self.migrations.get(addon, {}).get("module", {}).pop(version, None) + + +_get_files._original_method = MigrationManager._get_files +MigrationManager._get_files = _get_files diff --git a/openupgrade_framework/odoo_patch/odoo/modules/module_graph.py b/openupgrade_framework/odoo_patch/odoo/modules/module_graph.py new file mode 100644 index 000000000000..8c04d26931ae --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/modules/module_graph.py @@ -0,0 +1,26 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import os + +import odoo +from odoo.modules.module_graph import ModuleGraph + + +def _update_from_database(self, *args, **kwargs) -> None: + """Prevent reloading of demo data from the new version on major upgrade""" + ModuleGraph._update_from_database._original_method(self, *args, **kwargs) + + if os.environ.get("OPENUPGRADE_USE_DEMO", "") == "yes": + return + if ( + "base" in self._modules + and self._modules["base"].demo + and self._modules["base"].installed_version < odoo.release.major_version + ): + self._cr.execute("UPDATE ir_module_module SET demo = false") + for module in self._modules.values(): + module.demo = False + + +_update_from_database._original_method = ModuleGraph._update_from_database +ModuleGraph._update_from_database = _update_from_database diff --git a/openupgrade_framework/odoo_patch/odoo/orm/__init__.py b/openupgrade_framework/odoo_patch/odoo/orm/__init__.py new file mode 100644 index 000000000000..0650744f6bc6 --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/orm/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/openupgrade_framework/odoo_patch/odoo/orm/models.py b/openupgrade_framework/odoo_patch/odoo/orm/models.py new file mode 100644 index 000000000000..314147eae39f --- /dev/null +++ b/openupgrade_framework/odoo_patch/odoo/orm/models.py @@ -0,0 +1,42 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import logging +from uuid import uuid4 + +from odoo.models import BaseModel + +from odoo.addons.base.models.ir_model import MODULE_UNINSTALL_FLAG + +_logger = logging.getLogger(__name__) + + +def unlink(self): + """Don't break on unlink of obsolete records + when called from ir.model::_process_end() + + This only adapts the base unlink method. If overrides of this method + on individual models give problems, add patches for those as well. + """ + if not self.env.context.get(MODULE_UNINSTALL_FLAG): + return BaseModel.unlink._original_method(self) + savepoint = str(uuid4) + try: + self.env.cr.execute( # pylint: disable=sql-injection + f'SAVEPOINT "{savepoint}"' + ) + return BaseModel.unlink._original_method(self) + except Exception as e: + self.env.cr.execute( # pylint: disable=sql-injection + f'ROLLBACK TO SAVEPOINT "{savepoint}"' + ) + _logger.warning( + "Could not delete obsolete record with ids %s of model %s", + self.ids, + self._name, + exc_info=e, + ) + return False + + +unlink._original_method = BaseModel.unlink +BaseModel.unlink = unlink diff --git a/openupgrade_framework/pyproject.toml b/openupgrade_framework/pyproject.toml new file mode 100644 index 000000000000..4231d0cccb3d --- /dev/null +++ b/openupgrade_framework/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/openupgrade_framework/readme/CONFIGURE.md b/openupgrade_framework/readme/CONFIGURE.md new file mode 100644 index 000000000000..13ef1e5b0946 --- /dev/null +++ b/openupgrade_framework/readme/CONFIGURE.md @@ -0,0 +1,16 @@ +- call your odoo instance with the option + `--load=base,web,openupgrade_framework` + +or + +- add the key to your configuration file: + +``` shell +[options] +server_wide_modules = web,openupgrade_framework +``` + +When you load the module in either way of these ways, and you have the +openupgrade_scripts module in your addons path available, the +--upgrade-path option of Odoo will be set automatically to the location +of the OpenUpgrade migration scripts. diff --git a/openupgrade_framework/readme/CONTRIBUTORS.md b/openupgrade_framework/readme/CONTRIBUTORS.md new file mode 100644 index 000000000000..a108409b034c --- /dev/null +++ b/openupgrade_framework/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Stefan Rijnhart \<\> +- Sylvain LE GAL \<\> diff --git a/openupgrade_framework/readme/CREDITS.md b/openupgrade_framework/readme/CREDITS.md new file mode 100644 index 000000000000..fa515612942e --- /dev/null +++ b/openupgrade_framework/readme/CREDITS.md @@ -0,0 +1,4 @@ +Many developers have contributed to the OpenUpgrade framework in its +previous incarnation. Their original contributions may no longer needed, +or they are no longer recognizable in their current form but OpenUpgrade +would not have existed at this point without them. diff --git a/openupgrade_framework/readme/DESCRIPTION.md b/openupgrade_framework/readme/DESCRIPTION.md new file mode 100644 index 000000000000..71cb68ac0b99 --- /dev/null +++ b/openupgrade_framework/readme/DESCRIPTION.md @@ -0,0 +1,29 @@ +This module is a technical module that contains a number of +monkeypatches to improve the behaviour of Odoo when migrating your +database using the OpenUpgrade migration scripts: + +- Prevent dropping columns or tables in the database when fields or + models are obsoleted in the Odoo data model of the target release. + After the migration, you can review and delete unused database tables + and columns using database_cleanup. See + +- When data records are deleted during the migration (such as views or + other system records), this is done in a secure mode. If the deletion + fails because of some unforeseen dependency, the deletion will be + cancelled and a message is logged, after which the migration + continues. +- Prevent a number of log messages that do not apply when using + OpenUpgrade. +- Suppress log messages about failed view validation, which are to be + expected during a migration. +- Run migration scripts for modules that are installed as new + dependencies of upgraded modules (when there are such scripts for + those particular modules) +- Production databases generated with demo data, will be transformed to + non-demo ones. If you want to avoid that, you have to pass through the + environment variable OPENUPGRADE_USE_DEMO, the value "yes". + +For detailed documentation see: + +- +- diff --git a/openupgrade_framework/readme/DEVELOP.md b/openupgrade_framework/readme/DEVELOP.md new file mode 100644 index 000000000000..9d4c548862be --- /dev/null +++ b/openupgrade_framework/readme/DEVELOP.md @@ -0,0 +1,50 @@ +The odoo_patch folder contains python files in a tree that mimicks the +folder tree of the Odoo project. It contains a number of monkey patches +to improve the migration of an Odoo database between two major versions. + +So far, we are able to make everything work without overwriting large +blocks of code, but if larger patches need to be added, please use the +method described below: + +To see the patches added, you can use `meld` tools: + +`meld PATH_TO_ODOO_FOLDER/odoo/ PATH_TO_OPENUPGRADE_FRAMEWORK_MODULE/odoo_patch` + +To make more easy the diff analysis : + +- Make sure the python files has the same path as the original one. +- Keep the same indentation as the original file. (using `if True:` if + required) +- Add the following two lines at the beginning of your file, to avoid + flake8 / pylint errors + +``` python +# flake8: noqa +# pylint: skip-file +``` + +- When you want to change the code. add the following tags: + +For an addition: + +``` python +# +some code... +# +``` + +For a change: + +``` python +# +some code... +# +``` + +For a removal: + +``` python +# +# Comment the code, instead of removing it. +# +``` diff --git a/openupgrade_framework/readme/INSTALL.md b/openupgrade_framework/readme/INSTALL.md new file mode 100644 index 000000000000..471a1d162ce1 --- /dev/null +++ b/openupgrade_framework/readme/INSTALL.md @@ -0,0 +1,2 @@ +This module does not need to be installed on a database. It simply needs +to be available via your `addons-path`. diff --git a/openupgrade_framework/static/description/banner.png b/openupgrade_framework/static/description/banner.png new file mode 100644 index 000000000000..9dd67ec04cf4 Binary files /dev/null and b/openupgrade_framework/static/description/banner.png differ diff --git a/openupgrade_framework/static/description/icon.png b/openupgrade_framework/static/description/icon.png new file mode 100644 index 000000000000..3a0328b516c4 Binary files /dev/null and b/openupgrade_framework/static/description/icon.png differ diff --git a/openupgrade_framework/static/description/index.html b/openupgrade_framework/static/description/index.html new file mode 100644 index 000000000000..543f2d15a681 --- /dev/null +++ b/openupgrade_framework/static/description/index.html @@ -0,0 +1,543 @@ + + + + + +Openupgrade Framework + + + +
+ + + +Odoo Community Association + +
+

Openupgrade Framework

+ +

Beta License: AGPL-3 OCA/OpenUpgrade Translate me on Weblate Try me on Runboat

+

This module is a technical module that contains a number of +monkeypatches to improve the behaviour of Odoo when migrating your +database using the OpenUpgrade migration scripts:

+
    +
  • Prevent dropping columns or tables in the database when fields or +models are obsoleted in the Odoo data model of the target release. +After the migration, you can review and delete unused database tables +and columns using database_cleanup. See +https://odoo-community.org/shop/product/database-cleanup-918
    • +
  • When data records are deleted during the migration (such as views or +other system records), this is done in a secure mode. If the deletion +fails because of some unforeseen dependency, the deletion will be +cancelled and a message is logged, after which the migration +continues.
    • +
  • Prevent a number of log messages that do not apply when using +OpenUpgrade.
    • +
  • Suppress log messages about failed view validation, which are to be +expected during a migration.
    • +
  • Run migration scripts for modules that are installed as new +dependencies of upgraded modules (when there are such scripts for +those particular modules)
    • +
  • Production databases generated with demo data, will be transformed to +non-demo ones. If you want to avoid that, you have to pass through the +environment variable OPENUPGRADE_USE_DEMO, the value “yes”.
    • +
+

For detailed documentation see:

+ +

Table of contents

+ +
+

Installation

+

This module does not need to be installed on a database. It simply needs +to be available via your addons-path.

+
+
+

Configuration

+
    +
  • call your odoo instance with the option +--load=base,web,openupgrade_framework
    • +
+

or

+
    +
  • add the key to your configuration file:
    • +
+
+[options]
+server_wide_modules = web,openupgrade_framework
+
+

When you load the module in either way of these ways, and you have the +openupgrade_scripts module in your addons path available, the +–upgrade-path option of Odoo will be set automatically to the location +of the OpenUpgrade migration scripts.

+
+
+

Development

+

The odoo_patch folder contains python files in a tree that mimicks the +folder tree of the Odoo project. It contains a number of monkey patches +to improve the migration of an Odoo database between two major versions.

+

So far, we are able to make everything work without overwriting large +blocks of code, but if larger patches need to be added, please use the +method described below:

+

To see the patches added, you can use meld tools:

+

meld PATH_TO_ODOO_FOLDER/odoo/ PATH_TO_OPENUPGRADE_FRAMEWORK_MODULE/odoo_patch

+

To make more easy the diff analysis :

+
    +
  • Make sure the python files has the same path as the original one.
    • +
  • Keep the same indentation as the original file. (using if True: if +required)
    • +
  • Add the following two lines at the beginning of your file, to avoid +flake8 / pylint errors
    • +
+
+# flake8: noqa
+# pylint: skip-file
+
+
    +
  • When you want to change the code. add the following tags:
    • +
+

For an addition:

+
+# <OpenUpgrade:ADD>
+some code...
+# </OpenUpgrade>
+
+

For a change:

+
+# <OpenUpgrade:CHANGE>
+some code...
+# </OpenUpgrade>
+
+

For a removal:

+
+# <OpenUpgrade:REMOVE>
+# Comment the code, instead of removing it.
+# </OpenUpgrade>
+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Therp BV
    • +
  • Opener B.V.
    • +
  • GRAP
    • +
  • Hunki Enterprises BV
    • +
+
+
+

Contributors

+ +
+
+

Other credits

+

Many developers have contributed to the OpenUpgrade framework in its +previous incarnation. Their original contributions may no longer needed, +or they are no longer recognizable in their current form but OpenUpgrade +would not have existed at this point without them.

+
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainers:

+

legalsylvain StefanRijnhart hbrunn

+

This module is part of the OCA/OpenUpgrade project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+
+ + diff --git a/openupgrade_framework/tests/__init__.py b/openupgrade_framework/tests/__init__.py new file mode 100644 index 000000000000..f0425e721034 --- /dev/null +++ b/openupgrade_framework/tests/__init__.py @@ -0,0 +1 @@ +from . import test_openupgrade_framework diff --git a/openupgrade_framework/tests/test_openupgrade_framework.py b/openupgrade_framework/tests/test_openupgrade_framework.py new file mode 100644 index 000000000000..4a763142f32f --- /dev/null +++ b/openupgrade_framework/tests/test_openupgrade_framework.py @@ -0,0 +1,35 @@ +from odoo.tests.common import TransactionCase +from odoo.tools.misc import mute_logger + +from odoo.addons.base.models.ir_model import MODULE_UNINSTALL_FLAG + + +class TestOpenupgradeFramework(TransactionCase): + def test_01_delete_undeletable_record(self): + """ + Test that Odoo doesn't crash when deleting records that can't be deleted + during module upgrade + """ + with ( + mute_logger("odoo.sql_db"), + self.assertLogs( + "odoo.addons.openupgrade_framework.odoo_patch.odoo.orm.models" + ), + ): + self.env.ref("base.partner_admin").with_context( + **{MODULE_UNINSTALL_FLAG: True} + ).unlink() + + def test_02_invalid_view(self): + """ + Test that we patch away fatal view errors and log the problem + """ + with self.assertLogs( + "odoo.addons.openupgrade_framework.odoo_patch.odoo.addons.base.models.ir_ui_view" + ): + self.env["ir.ui.view"].create( + { + "model": "res.partner", + "arch": '
', + } + ) diff --git a/openupgrade_scripts/README.rst b/openupgrade_scripts/README.rst new file mode 100644 index 000000000000..6bf11e20eb57 --- /dev/null +++ b/openupgrade_scripts/README.rst @@ -0,0 +1,92 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +=================== +Openupgrade Scripts +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:2029125d9bf0d1f2c4246f5bd141a1f6740cfd74b48f4218d6c57621d9868df7 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2FOpenUpgrade-lightgray.png?logo=github + :target: https://github.com/OCA/OpenUpgrade/tree/20.0/openupgrade_scripts + :alt: OCA/OpenUpgrade +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/OpenUpgrade-20-0/OpenUpgrade-20-0-openupgrade_scripts + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/OpenUpgrade&target_branch=20.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module is a containers of migration script to migrate from 19.0 to +20.0 version. + +**Table of contents** + +.. contents:: + :local: + +Installation +============ + +This module does not need to be installed on a database. It simply needs +to be available via your ``addons-path``. + +Configuration +============= + +- call your odoo instance with the option + ``--upgrade-path=/PATH_TO_openupgrade_scripts_MODULE/scripts/`` + +or + +- add the key to your configuration file: + +.. code:: shell + + [options] + upgrade_path = /PATH_TO_openupgrade_scripts_MODULE/scripts/ + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/OpenUpgrade `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/openupgrade_scripts/__init__.py b/openupgrade_scripts/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openupgrade_scripts/__manifest__.py b/openupgrade_scripts/__manifest__.py new file mode 100644 index 000000000000..861907aef0ae --- /dev/null +++ b/openupgrade_scripts/__manifest__.py @@ -0,0 +1,16 @@ +# Copyright Odoo Community Association (OCA) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +{ + "name": "Openupgrade Scripts", + "summary": "Module that contains all the migration analysis files " + "and scripts to migrate Odoo SA modules.", + "author": "Odoo Community Association (OCA)", + "website": "https://github.com/OCA/OpenUpgrade", + "category": "Migration", + "version": "20.0.1.0.0", + "license": "AGPL-3", + "depends": ["base"], + "images": ["static/description/banner.jpg"], + "external_dependencies": {"python": ["openupgradelib"]}, + "installable": True, +} diff --git a/openupgrade_scripts/apriori.py b/openupgrade_scripts/apriori.py new file mode 100644 index 000000000000..ad2269309acc --- /dev/null +++ b/openupgrade_scripts/apriori.py @@ -0,0 +1,30 @@ +"""Encode any known changes to the database here +to help the matching process +""" + +# Renamed modules is a mapping from old module name to new module name +renamed_modules = { + # odoo + # odoo/enterprise + # OCA/... +} + +# Merged modules contain a mapping from old module names to other, +# preexisting module names +merged_modules = { + # odoo + # odoo/enterprise + # OCA/... +} + +# only used here for upgrade_analysis +renamed_models = { + # odoo + # OCA/... +} + +# only used here for upgrade_analysis +merged_models = { + # odoo + # OCA/... +} diff --git a/openupgrade_scripts/pyproject.toml b/openupgrade_scripts/pyproject.toml new file mode 100644 index 000000000000..4231d0cccb3d --- /dev/null +++ b/openupgrade_scripts/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/openupgrade_scripts/readme/CONFIGURE.md b/openupgrade_scripts/readme/CONFIGURE.md new file mode 100644 index 000000000000..102ae2481b2f --- /dev/null +++ b/openupgrade_scripts/readme/CONFIGURE.md @@ -0,0 +1,11 @@ +- call your odoo instance with the option + `--upgrade-path=/PATH_TO_openupgrade_scripts_MODULE/scripts/` + +or + +- add the key to your configuration file: + +``` shell +[options] +upgrade_path = /PATH_TO_openupgrade_scripts_MODULE/scripts/ +``` diff --git a/openupgrade_scripts/readme/DESCRIPTION.md b/openupgrade_scripts/readme/DESCRIPTION.md new file mode 100644 index 000000000000..3ad1160de8a0 --- /dev/null +++ b/openupgrade_scripts/readme/DESCRIPTION.md @@ -0,0 +1,2 @@ +This module is a containers of migration script to migrate from 19.0 to +20.0 version. diff --git a/openupgrade_scripts/readme/INSTALL.md b/openupgrade_scripts/readme/INSTALL.md new file mode 100644 index 000000000000..471a1d162ce1 --- /dev/null +++ b/openupgrade_scripts/readme/INSTALL.md @@ -0,0 +1,2 @@ +This module does not need to be installed on a database. It simply needs +to be available via your `addons-path`. diff --git a/openupgrade_scripts/static/description/banner.png b/openupgrade_scripts/static/description/banner.png new file mode 100644 index 000000000000..9dd67ec04cf4 Binary files /dev/null and b/openupgrade_scripts/static/description/banner.png differ diff --git a/openupgrade_scripts/static/description/icon.png b/openupgrade_scripts/static/description/icon.png new file mode 100644 index 000000000000..3a0328b516c4 Binary files /dev/null and b/openupgrade_scripts/static/description/icon.png differ diff --git a/openupgrade_scripts/static/description/index.html b/openupgrade_scripts/static/description/index.html new file mode 100644 index 000000000000..c077c70b31ac --- /dev/null +++ b/openupgrade_scripts/static/description/index.html @@ -0,0 +1,438 @@ + + + + + +Openupgrade Scripts + + + +
+ + + +Odoo Community Association + +
+

Openupgrade Scripts

+ +

Beta License: AGPL-3 OCA/OpenUpgrade Translate me on Weblate Try me on Runboat

+

This module is a containers of migration script to migrate from 19.0 to +20.0 version.

+

Table of contents

+ +
+

Installation

+

This module does not need to be installed on a database. It simply needs +to be available via your addons-path.

+
+
+

Configuration

+
    +
  • call your odoo instance with the option +--upgrade-path=/PATH_TO_openupgrade_scripts_MODULE/scripts/
    • +
+

or

+
    +
  • add the key to your configuration file:
    • +
+
+[options]
+upgrade_path = /PATH_TO_openupgrade_scripts_MODULE/scripts/
+
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/OpenUpgrade project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+
+ + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..180fc49789ba --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +# generated from manifests external_dependencies +openupgradelib