diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..20d6fa4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: CI + +on: + push: + branches: + - unity-v3 + pull_request: + +concurrency: + group: excel_view-ci-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-latest + name: Server tests + + services: + redis-cache: + image: redis:alpine + ports: + - 13000:6379 + redis-queue: + image: redis:alpine + ports: + - 11000:6379 + mariadb: + image: mariadb:10.6 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install MariaDB client + run: | + sudo apt update + sudo apt-get install -y mariadb-client + + - name: Setup bench + run: | + pip install frappe-bench + bench init --skip-redis-config-generation --skip-assets --frappe-branch version-15 --python "$(which python)" ~/frappe-bench + mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" + mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" + + - name: Install app + working-directory: /home/runner/frappe-bench + run: | + bench get-app excel_view $GITHUB_WORKSPACE + bench new-site --db-root-password root --admin-password admin test_site + bench --site test_site install-app excel_view + env: + CI: "Yes" + + - name: Run tests + working-directory: /home/runner/frappe-bench + run: | + bench --site test_site set-config allow_tests true + bench --site test_site run-tests --app excel_view diff --git a/excel_view/api.py b/excel_view/api.py index ad860ee..0a88657 100644 --- a/excel_view/api.py +++ b/excel_view/api.py @@ -27,22 +27,26 @@ import frappe from frappe import _ + from excel_view.decorators import excel_whitelist # orjson: 5-10x faster JSON parsing/serialization than stdlib json. # Used for all our heavy payload parsing (bulk_fetch, smart_lookup, etc.) try: - import orjson as _json_lib - def _fast_parse(s): - if isinstance(s, (bytes, bytearray)): - return _json_lib.loads(s) - return _json_lib.loads(s.encode() if isinstance(s, str) else s) - def _fast_dumps(obj): - return _json_lib.dumps(obj).decode() + import orjson as _json_lib + + def _fast_parse(s): + if isinstance(s, bytes | bytearray): + return _json_lib.loads(s) + return _json_lib.loads(s.encode() if isinstance(s, str) else s) + + def _fast_dumps(obj): + return _json_lib.dumps(obj).decode() except ImportError: - import json as _json_lib - _fast_parse = _json_lib.loads - _fast_dumps = _json_lib.dumps + import json as _json_lib + + _fast_parse = _json_lib.loads + _fast_dumps = _json_lib.dumps def _check_doctype_permission(doctype, ptype="read"): @@ -57,19 +61,19 @@ def _check_doctype_permission(doctype, ptype="read"): #: Fields that exist on every DocType but are NOT in meta.fields — always valid. _SYSTEM_FIELDS: frozenset[str] = frozenset( - { - "name", - "owner", - "creation", - "modified", - "modified_by", - "docstatus", - "idx", - "_user_tags", - "_assign", - "_comments", - "_seen", - } + { + "name", + "owner", + "creation", + "modified", + "modified_by", + "docstatus", + "idx", + "_user_tags", + "_assign", + "_comments", + "_seen", + } ) #: Hard cap for frappe_aggregate SUM/AVG to avoid runaway DB scans. @@ -77,57 +81,61 @@ def _check_doctype_permission(doctype, ptype="read"): def _validate_fieldname(doctype: str, fieldname: str) -> None: - """ - Raise DoesNotExistError if *fieldname* is not a valid field on *doctype*. + """ + Raise DoesNotExistError if *fieldname* is not a valid field on *doctype*. - Checks system fields first (always allowed), then the live meta — which - includes custom fields added by any installed app. This makes the - validation fully dynamic without any hardcoded field lists. - """ - if fieldname in _SYSTEM_FIELDS: - return + Checks system fields first (always allowed), then the live meta — which + includes custom fields added by any installed app. This makes the + validation fully dynamic without any hardcoded field lists. + """ + if fieldname in _SYSTEM_FIELDS: + return - meta = frappe.get_meta(doctype) - valid_fields = {df.fieldname for df in meta.fields} + meta = frappe.get_meta(doctype) + valid_fields = {df.fieldname for df in meta.fields} - if fieldname not in valid_fields: - frappe.throw( - _("Field '{0}' does not exist on DocType '{1}'.").format(fieldname, doctype), - frappe.DoesNotExistError, - ) + if fieldname not in valid_fields: + frappe.throw( + _("Field '{0}' does not exist on DocType '{1}'.").format(fieldname, doctype), + frappe.DoesNotExistError, + ) _FILTER_OPS = frozenset({">", "<", ">=", "<=", "!=", "like", "not like", "in", "not in", "between"}) def _parse_filter_pairs( - fk1=None, fv1=None, - fk2=None, fv2=None, - fk3=None, fv3=None, - fk4=None, fv4=None, + fk1=None, + fv1=None, + fk2=None, + fv2=None, + fk3=None, + fv3=None, + fk4=None, + fv4=None, ) -> list: - """ - Build a Frappe filter list from up to four SUMIF-style key/value pairs. - - Supports both exact-match and operator-style keys: - Exact: fk="ev_sales_person", fv="Arjun" → ["ev_sales_person", "=", "Arjun"] - Operator: fk="transaction_date >=", fv="2026-03-01" → ["transaction_date", ">=", "2026-03-01"] - - Keys/values that are None or empty are silently skipped. - Returns a list of [field, op, value] triples (frappe.get_list accepts this format). - """ - filters = [] - for key, val in ((fk1, fv1), (fk2, fv2), (fk3, fv3), (fk4, fv4)): - if not key or val is None or val == "": - continue - key = str(key).strip() - # Detect trailing operator: "transaction_date >=" → field="transaction_date", op=">=" - parts = key.rsplit(" ", 1) - if len(parts) == 2 and parts[1].lower() in _FILTER_OPS: - filters.append([parts[0], parts[1], val]) - else: - filters.append([key, "=", val]) - return filters + """ + Build a Frappe filter list from up to four SUMIF-style key/value pairs. + + Supports both exact-match and operator-style keys: + Exact: fk="ev_sales_person", fv="Arjun" → ["ev_sales_person", "=", "Arjun"] + Operator: fk="transaction_date >=", fv="2026-03-01" → ["transaction_date", ">=", "2026-03-01"] + + Keys/values that are None or empty are silently skipped. + Returns a list of [field, op, value] triples (frappe.get_list accepts this format). + """ + filters = [] + for key, val in ((fk1, fv1), (fk2, fv2), (fk3, fv3), (fk4, fv4)): + if not key or val is None or val == "": + continue + key = str(key).strip() + # Detect trailing operator: "transaction_date >=" → field="transaction_date", op=">=" + parts = key.rsplit(" ", 1) + if len(parts) == 2 and parts[1].lower() in _FILTER_OPS: + filters.append([parts[0], parts[1], val]) + else: + filters.append([key, "=", val]) + return filters # ── Child Table — Inline Expand (V3.5) ─────────────────────────────────────── @@ -138,46 +146,47 @@ def _parse_filter_pairs( # so _server_messages never reach the browser — all error display is handled by # the JS EVModal/toast system. + def _make_ev_error(exc: Exception) -> dict: - """ - Serialise *exc* into the structured error dict that the front-end - _ev_call() helper recognises and routes through EVModal. - - Priority for the user-facing message: - 1. frappe.message_log — populated by frappe.throw(); contains the - translated, HTML-formatted message that Frappe normally shows in its - own dialog. We strip the HTML and use the plain text. - 2. str(exc) — raw Python exception message; acceptable but - may contain internal identifiers (e.g. MandatoryError format). - - IMPORTANT: call this *before* frappe.message_log.clear() so the log is - still populated. - """ - exc_type = type(exc).__name__.split(".")[-1] - - msg: str | None = None - - # 1 — user-friendly message from frappe.message_log - if frappe.message_log: - try: - entry = frappe.message_log[-1] - if isinstance(entry, dict): - msg = entry.get("message") or entry.get("title") - else: - parsed = (_fast_parse(entry) if isinstance(entry, str) else (entry or [])) or {} - msg = parsed.get("message") or parsed.get("title") - except Exception: - pass - - # 2 — fallback to exception string - if not msg: - msg = str(exc) if exc else frappe._("An unexpected error occurred.") - - return { - "_ev_error": True, - "message": frappe.utils.strip_html(msg), - "exc_type": exc_type, - } + """ + Serialise *exc* into the structured error dict that the front-end + _ev_call() helper recognises and routes through EVModal. + + Priority for the user-facing message: + 1. frappe.message_log — populated by frappe.throw(); contains the + translated, HTML-formatted message that Frappe normally shows in its + own dialog. We strip the HTML and use the plain text. + 2. str(exc) — raw Python exception message; acceptable but + may contain internal identifiers (e.g. MandatoryError format). + + IMPORTANT: call this *before* frappe.message_log.clear() so the log is + still populated. + """ + exc_type = type(exc).__name__.split(".")[-1] + + msg: str | None = None + + # 1 — user-friendly message from frappe.message_log + if frappe.message_log: + try: + entry = frappe.message_log[-1] + if isinstance(entry, dict): + msg = entry.get("message") or entry.get("title") + else: + parsed = (_fast_parse(entry) if isinstance(entry, str) else (entry or [])) or {} + msg = parsed.get("message") or parsed.get("title") + except Exception: + pass + + # 2 — fallback to exception string + if not msg: + msg = str(exc) if exc else frappe._("An unexpected error occurred.") + + return { + "_ev_error": True, + "message": frappe.utils.strip_html(msg), + "exc_type": exc_type, + } # @@ -193,395 +202,402 @@ def _make_ev_error(exc: Exception) -> dict: # - All fieldnames are validated against live meta — no raw SQL injection. #: Field types that carry no storable value (sections, columns, HTML, etc.) -_NO_VALUE_TYPES: frozenset[str] = frozenset({ - "Section Break", "Column Break", "Tab Break", "HTML", "Fold", "Heading", - "Button", "Image", "Barcode", "Geolocation", "Table", "Table MultiSelect", -}) +_NO_VALUE_TYPES: frozenset[str] = frozenset( + { + "Section Break", + "Column Break", + "Tab Break", + "HTML", + "Fold", + "Heading", + "Button", + "Image", + "Barcode", + "Geolocation", + "Table", + "Table MultiSelect", + } +) def _get_child_table_df(parent_meta, child_fieldname: str): - """ - Return the field descriptor for *child_fieldname* on *parent_meta*. - Raises ValidationError if the field does not exist or is not a Table type. - """ - df = next( - (f for f in parent_meta.fields - if f.fieldname == child_fieldname and f.fieldtype in ("Table", "Table MultiSelect")), - None, - ) - if not df: - frappe.throw( - _("'{0}' is not a valid child table field on {1}.").format( - child_fieldname, parent_meta.name - ) - ) - return df + """ + Return the field descriptor for *child_fieldname* on *parent_meta*. + Raises ValidationError if the field does not exist or is not a Table type. + """ + df = next( + ( + f + for f in parent_meta.fields + if f.fieldname == child_fieldname and f.fieldtype in ("Table", "Table MultiSelect") + ), + None, + ) + if not df: + frappe.throw( + _("'{0}' is not a valid child table field on {1}.").format(child_fieldname, parent_meta.name) + ) + return df def _child_col_defs(child_meta) -> list[dict]: - """ - Return a JSON-serialisable list of column definitions for the child DocType. - Only includes fields with a storable value (excludes Section Break etc.). - The 'name' system field is prepended so each row has a stable identifier. - `allow_on_submit` mirrors Frappe's perm.js logic: tells the UI whether this - field may be edited on an already-submitted parent document. - """ - cols = [] - for f in child_meta.fields: - if f.fieldtype in _NO_VALUE_TYPES: - continue - cols.append({ - "fieldname": f.fieldname, - "label": _(f.label or f.fieldname), - "fieldtype": f.fieldtype, - "options": f.options or "", - "reqd": bool(f.reqd), - "read_only": bool(f.read_only), - "hidden": bool(f.hidden), - "in_list_view": bool(f.in_list_view), - "default": f.default or "", - "allow_on_submit": bool(f.allow_on_submit), # V3.5: submitted-doc edit guard - }) - return cols + """ + Return a JSON-serialisable list of column definitions for the child DocType. + Only includes fields with a storable value (excludes Section Break etc.). + The 'name' system field is prepended so each row has a stable identifier. + `allow_on_submit` mirrors Frappe's perm.js logic: tells the UI whether this + field may be edited on an already-submitted parent document. + """ + cols = [] + for f in child_meta.fields: + if f.fieldtype in _NO_VALUE_TYPES: + continue + cols.append( + { + "fieldname": f.fieldname, + "label": _(f.label or f.fieldname), + "fieldtype": f.fieldtype, + "options": f.options or "", + "reqd": bool(f.reqd), + "read_only": bool(f.read_only), + "hidden": bool(f.hidden), + "in_list_view": bool(f.in_list_view), + "default": f.default or "", + "allow_on_submit": bool(f.allow_on_submit), # V3.5: submitted-doc edit guard + } + ) + return cols @frappe.whitelist() def get_child_rows(doctype: str, parent_name: str, child_fieldname: str) -> dict: - """ - Return all rows of one child table for a single parent document. - - Args: - doctype: Parent DocType (e.g. "Sales Order") - parent_name: Name of the parent document (e.g. "SAL-ORD-2026-00001") - child_fieldname: Fieldname of the child table on the parent (e.g. "items") - - Returns: - { - "child_doctype": "Sales Order Item", - "fields": [ { fieldname, label, fieldtype, options, reqd, ... }, ... ], - "rows": [ { name, idx, fieldname: value, ... }, ... ] - } - - Security: READ on parent DocType is required. - """ - frappe.has_permission(doctype, "read", parent_name, throw=True) - - parent_meta = frappe.get_meta(doctype) - df = _get_child_table_df(parent_meta, child_fieldname) - child_doctype: str = df.options - - # V3.5 — Submitted-doc edit guard: fetch parent docstatus in one cheap query - parent_docstatus: int = frappe.db.get_value(doctype, parent_name, "docstatus") or 0 - - child_meta = frappe.get_meta(child_doctype) - col_defs = _child_col_defs(child_meta) - - # Build the field list for the DB query — system fields + all value-bearing fields - fetch_fields = ["name", "idx"] + [c["fieldname"] for c in col_defs] - # Deduplicate while preserving order (name/idx might appear in col_defs too) - seen: set[str] = set() - safe_fields: list[str] = [] - for f in fetch_fields: - if f not in seen: - seen.add(f) - safe_fields.append(f) - - rows = frappe.get_all( - child_doctype, - filters={ - "parent": parent_name, - "parentfield": child_fieldname, - "parenttype": doctype, - }, - fields=safe_fields, - order_by="idx asc", - limit=0, - ) - - return { - "child_doctype": child_doctype, - "fields": col_defs, - "rows": [dict(r) for r in rows], - # V3.5 — submitted-doc awareness - "parent_docstatus": parent_docstatus, - "table_allow_on_submit": bool(df.allow_on_submit), # controls add/delete - } + """ + Return all rows of one child table for a single parent document. + + Args: + doctype: Parent DocType (e.g. "Sales Order") + parent_name: Name of the parent document (e.g. "SAL-ORD-2026-00001") + child_fieldname: Fieldname of the child table on the parent (e.g. "items") + + Returns: + { + "child_doctype": "Sales Order Item", + "fields": [ { fieldname, label, fieldtype, options, reqd, ... }, ... ], + "rows": [ { name, idx, fieldname: value, ... }, ... ] + } + + Security: READ on parent DocType is required. + """ + frappe.has_permission(doctype, "read", parent_name, throw=True) + + parent_meta = frappe.get_meta(doctype) + df = _get_child_table_df(parent_meta, child_fieldname) + child_doctype: str = df.options + + # V3.5 — Submitted-doc edit guard: fetch parent docstatus in one cheap query + parent_docstatus: int = frappe.db.get_value(doctype, parent_name, "docstatus") or 0 + + child_meta = frappe.get_meta(child_doctype) + col_defs = _child_col_defs(child_meta) + + # Build the field list for the DB query — system fields + all value-bearing fields + fetch_fields = ["name", "idx"] + [c["fieldname"] for c in col_defs] + # Deduplicate while preserving order (name/idx might appear in col_defs too) + seen: set[str] = set() + safe_fields: list[str] = [] + for f in fetch_fields: + if f not in seen: + seen.add(f) + safe_fields.append(f) + + rows = frappe.get_all( + child_doctype, + filters={ + "parent": parent_name, + "parentfield": child_fieldname, + "parenttype": doctype, + }, + fields=safe_fields, + order_by="idx asc", + limit=0, + ) + + return { + "child_doctype": child_doctype, + "fields": col_defs, + "rows": [dict(r) for r in rows], + # V3.5 — submitted-doc awareness + "parent_docstatus": parent_docstatus, + "table_allow_on_submit": bool(df.allow_on_submit), # controls add/delete + } @frappe.whitelist() def save_child_row( - doctype: str, - parent_name: str, - child_fieldname: str, - child_name: str, - fields: str, + doctype: str, + parent_name: str, + child_fieldname: str, + child_name: str, + fields: str, ) -> dict: - """ - Update one child row via the full Frappe ORM (parent_doc.save()). - - Runs all controller hooks (validate, before_save, on_change, after_save) - — same as saving from the Frappe form UI. - - Security: WRITE on parent DocType is required. - """ - try: - frappe.has_permission(doctype, "write", parent_name, throw=True) - - fields_dict: dict = (_fast_parse(fields) if isinstance(fields, str) else fields) - if not fields_dict: - return {"name": child_name} - - parent_meta = frappe.get_meta(doctype) - df = _get_child_table_df(parent_meta, child_fieldname) - child_doctype: str = df.options - - child_meta = frappe.get_meta(child_doctype) - valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS - for fn in fields_dict: - if fn not in valid_fields: - frappe.throw( - _("Field '{0}' does not exist on {1}.").format(fn, child_doctype) - ) - - parent_doc = frappe.get_doc(doctype, parent_name) - target_child = next( - (c for c in (parent_doc.get(child_fieldname) or []) if c.name == child_name), - None, - ) - if not target_child: - frappe.throw( - _("Child row '{0}' not found in {1}.{2}").format( - child_name, parent_name, child_fieldname - ) - ) - - for fn, val in fields_dict.items(): - target_child.set(fn, val) - - parent_doc.save() - frappe.db.commit() - return {"name": child_name} - - except Exception as exc: - ev_err = _make_ev_error(exc) - frappe.message_log.clear() - frappe.db.rollback() - return ev_err + """ + Update one child row via the full Frappe ORM (parent_doc.save()). + + Runs all controller hooks (validate, before_save, on_change, after_save) + — same as saving from the Frappe form UI. + + Security: WRITE on parent DocType is required. + """ + try: + frappe.has_permission(doctype, "write", parent_name, throw=True) + + fields_dict: dict = _fast_parse(fields) if isinstance(fields, str) else fields + if not fields_dict: + return {"name": child_name} + + parent_meta = frappe.get_meta(doctype) + df = _get_child_table_df(parent_meta, child_fieldname) + child_doctype: str = df.options + + child_meta = frappe.get_meta(child_doctype) + valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS + for fn in fields_dict: + if fn not in valid_fields: + frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) + + parent_doc = frappe.get_doc(doctype, parent_name) + target_child = next( + (c for c in (parent_doc.get(child_fieldname) or []) if c.name == child_name), + None, + ) + if not target_child: + frappe.throw( + _("Child row '{0}' not found in {1}.{2}").format(child_name, parent_name, child_fieldname) + ) + + for fn, val in fields_dict.items(): + target_child.set(fn, val) + + parent_doc.save() + frappe.db.commit() + return {"name": child_name} + + except Exception as exc: + ev_err = _make_ev_error(exc) + frappe.message_log.clear() + frappe.db.rollback() + return ev_err @frappe.whitelist() def insert_child_row( - doctype: str, - parent_name: str, - child_fieldname: str, - fields: str, + doctype: str, + parent_name: str, + child_fieldname: str, + fields: str, ) -> dict: - """ - Append a new child row via the full Frappe ORM (parent_doc.append + save). + """ + Append a new child row via the full Frappe ORM (parent_doc.append + save). - Runs all controller hooks — identical to adding a row from the Frappe form UI. + Runs all controller hooks — identical to adding a row from the Frappe form UI. - Returns: { "name": new_child_name, "idx": new_idx } - Security: WRITE on parent DocType is required. - """ - try: - frappe.has_permission(doctype, "write", parent_name, throw=True) + Returns: { "name": new_child_name, "idx": new_idx } + Security: WRITE on parent DocType is required. + """ + try: + frappe.has_permission(doctype, "write", parent_name, throw=True) - fields_dict: dict = (_fast_parse(fields) if isinstance(fields, str) else (fields or {})) + fields_dict: dict = _fast_parse(fields) if isinstance(fields, str) else (fields or {}) - parent_meta = frappe.get_meta(doctype) - df = _get_child_table_df(parent_meta, child_fieldname) - child_doctype: str = df.options + parent_meta = frappe.get_meta(doctype) + df = _get_child_table_df(parent_meta, child_fieldname) + child_doctype: str = df.options - if fields_dict: - child_meta = frappe.get_meta(child_doctype) - valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS - for fn in fields_dict: - if fn not in valid_fields: - frappe.throw( - _("Field '{0}' does not exist on {1}.").format(fn, child_doctype) - ) + if fields_dict: + child_meta = frappe.get_meta(child_doctype) + valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS + for fn in fields_dict: + if fn not in valid_fields: + frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) - parent_doc = frappe.get_doc(doctype, parent_name) - child_row = parent_doc.append(child_fieldname, fields_dict) - parent_doc.save() - frappe.db.commit() - return {"name": child_row.name, "idx": child_row.idx} + parent_doc = frappe.get_doc(doctype, parent_name) + child_row = parent_doc.append(child_fieldname, fields_dict) + parent_doc.save() + frappe.db.commit() + return {"name": child_row.name, "idx": child_row.idx} - except Exception as exc: - ev_err = _make_ev_error(exc) - frappe.message_log.clear() - frappe.db.rollback() - return ev_err + except Exception as exc: + ev_err = _make_ev_error(exc) + frappe.message_log.clear() + frappe.db.rollback() + return ev_err @frappe.whitelist() def delete_child_row( - doctype: str, - parent_name: str, - child_fieldname: str, - child_name: str, + doctype: str, + parent_name: str, + child_fieldname: str, + child_name: str, ) -> dict: - """ - Remove a child row from a parent document via the full Frappe ORM. + """ + Remove a child row from a parent document via the full Frappe ORM. - The parent document is loaded, the matching child row is removed from the - in-memory list, and parent_doc.save() is called so all hooks run correctly - (e.g. total recalculation, on_change, after_save). + The parent document is loaded, the matching child row is removed from the + in-memory list, and parent_doc.save() is called so all hooks run correctly + (e.g. total recalculation, on_change, after_save). - Args: - doctype: Parent DocType - parent_name: Name of the parent document - child_fieldname: Child table fieldname on the parent - child_name: `name` of the child row to delete + Args: + doctype: Parent DocType + parent_name: Name of the parent document + child_fieldname: Child table fieldname on the parent + child_name: `name` of the child row to delete - Returns: - { "deleted": child_name } + Returns: + { "deleted": child_name } - Security: WRITE on parent DocType is required. - """ - try: - frappe.has_permission(doctype, "write", parent_name, throw=True) + Security: WRITE on parent DocType is required. + """ + try: + frappe.has_permission(doctype, "write", parent_name, throw=True) - parent_meta = frappe.get_meta(doctype) - _get_child_table_df(parent_meta, child_fieldname) + parent_meta = frappe.get_meta(doctype) + _get_child_table_df(parent_meta, child_fieldname) - parent_doc = frappe.get_doc(doctype, parent_name) - original_children = parent_doc.get(child_fieldname) or [] - filtered = [c for c in original_children if c.name != child_name] + parent_doc = frappe.get_doc(doctype, parent_name) + original_children = parent_doc.get(child_fieldname) or [] + filtered = [c for c in original_children if c.name != child_name] - if len(filtered) == len(original_children): - frappe.throw( - _("Child row '{0}' not found in {1}.{2}").format( - child_name, parent_name, child_fieldname - ) - ) + if len(filtered) == len(original_children): + frappe.throw( + _("Child row '{0}' not found in {1}.{2}").format(child_name, parent_name, child_fieldname) + ) - parent_doc.set(child_fieldname, filtered) - parent_doc.save() - frappe.db.commit() - return {"deleted": child_name} + parent_doc.set(child_fieldname, filtered) + parent_doc.save() + frappe.db.commit() + return {"deleted": child_name} - except Exception as exc: - ev_err = _make_ev_error(exc) # capture friendly msg BEFORE clearing log - frappe.message_log.clear() - frappe.db.rollback() - return ev_err + except Exception as exc: + ev_err = _make_ev_error(exc) # capture friendly msg BEFORE clearing log + frappe.message_log.clear() + frappe.db.rollback() + return ev_err @frappe.whitelist() def save_child_table_bulk( - doctype: str, - parent_name: str, - child_fieldname: str, - changes: str, + doctype: str, + parent_name: str, + child_fieldname: str, + changes: str, ) -> dict: - """ - Save ALL pending child-table changes in a single transaction. - - ``changes`` is a JSON object:: - - { - "updates": [{"name": "", "fields": {fn: val, ...}}, ...], - "inserts": [{"temp_name": "_new_ct_", "fields": {fn: val, ...}}, ...] - } - - One ``parent_doc.save()`` → one DB transaction → all ORM hooks run once. - Dramatically faster than N individual save calls (no N round-trips). - - Returns:: - - {"ok": true, "inserted": [{"temp_name": "...", "name": "...", "idx": N}, ...]} - - Security: WRITE on parent DocType required. - """ - try: - frappe.has_permission(doctype, "write", parent_name, throw=True) - - changes_dict: dict = (_fast_parse(changes) if isinstance(changes, str) else (changes or {})) - updates: list = changes_dict.get("updates") or [] - inserts: list = changes_dict.get("inserts") or [] - deletes: list = changes_dict.get("deletes") or [] - - if not updates and not inserts and not deletes: - return {"ok": True, "inserted": []} - - parent_meta = frappe.get_meta(doctype) - df = _get_child_table_df(parent_meta, child_fieldname) - child_doctype = df.options - child_meta = frappe.get_meta(child_doctype) - valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS - - # Validate all field names up-front — fail fast before touching the DB - for upd in updates: - for fn in (upd.get("fields") or {}): - if fn not in valid_fields: - frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) - for ins in inserts: - for fn in (ins.get("fields") or {}): - if fn not in valid_fields: - frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) - - parent_doc = frappe.get_doc(doctype, parent_name) - existing = {c.name: c for c in (parent_doc.get(child_fieldname) or [])} - - # Apply deletes — remove from the in-memory child list before save - # so the controller sees the final desired state (e.g. total % = 100) - delete_set = set(deletes) - if delete_set: - parent_doc.set(child_fieldname, [ - row for row in (parent_doc.get(child_fieldname) or []) - if row.name not in delete_set - ]) - existing = {c.name: c for c in (parent_doc.get(child_fieldname) or [])} - - # Apply updates - for upd in updates: - child_name = upd.get("name") - fields_dict = upd.get("fields") or {} - if not child_name or not fields_dict: - continue - if child_name not in existing: - frappe.throw(_("Child row '{0}' not found.").format(child_name)) - for fn, val in fields_dict.items(): - existing[child_name].set(fn, val) - - # Apply inserts - new_rows: list = [] - for ins in inserts: - fields_dict = ins.get("fields") or {} - new_row = parent_doc.append(child_fieldname, fields_dict) - new_rows.append((ins.get("temp_name", ""), new_row)) - - parent_doc.save() - - # For submitted docs (docstatus=1), Frappe's update_after_submit flow runs - # on_update_after_submit() AFTER db_update(). Any fields recalculated there - # (e.g. allocated_amount via calculate_contribution()) are updated in-memory - # but never flushed to DB. Force-persist them now. - if parent_doc.docstatus == 1: - for row in (parent_doc.get(child_fieldname) or []): - row.db_update() - - frappe.db.commit() - - return { - "ok": True, - "inserted": [ - {"temp_name": temp_name, "name": row.name, "idx": row.idx} - for temp_name, row in new_rows - ], - } - - except Exception as exc: - ev_err = _make_ev_error(exc) - frappe.message_log.clear() - frappe.db.rollback() - return ev_err + """ + Save ALL pending child-table changes in a single transaction. + + ``changes`` is a JSON object:: + + { + "updates": [{"name": "", "fields": {fn: val, ...}}, ...], + "inserts": [{"temp_name": "_new_ct_", "fields": {fn: val, ...}}, ...] + } + + One ``parent_doc.save()`` → one DB transaction → all ORM hooks run once. + Dramatically faster than N individual save calls (no N round-trips). + + Returns:: + + {"ok": true, "inserted": [{"temp_name": "...", "name": "...", "idx": N}, ...]} + + Security: WRITE on parent DocType required. + """ + try: + frappe.has_permission(doctype, "write", parent_name, throw=True) + + changes_dict: dict = _fast_parse(changes) if isinstance(changes, str) else (changes or {}) + updates: list = changes_dict.get("updates") or [] + inserts: list = changes_dict.get("inserts") or [] + deletes: list = changes_dict.get("deletes") or [] + + if not updates and not inserts and not deletes: + return {"ok": True, "inserted": []} + + parent_meta = frappe.get_meta(doctype) + df = _get_child_table_df(parent_meta, child_fieldname) + child_doctype = df.options + child_meta = frappe.get_meta(child_doctype) + valid_fields = {f.fieldname for f in child_meta.fields} | _SYSTEM_FIELDS + + # Validate all field names up-front — fail fast before touching the DB + for upd in updates: + for fn in upd.get("fields") or {}: + if fn not in valid_fields: + frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) + for ins in inserts: + for fn in ins.get("fields") or {}: + if fn not in valid_fields: + frappe.throw(_("Field '{0}' does not exist on {1}.").format(fn, child_doctype)) + + parent_doc = frappe.get_doc(doctype, parent_name) + existing = {c.name: c for c in (parent_doc.get(child_fieldname) or [])} + + # Apply deletes — remove from the in-memory child list before save + # so the controller sees the final desired state (e.g. total % = 100) + delete_set = set(deletes) + if delete_set: + parent_doc.set( + child_fieldname, + [row for row in (parent_doc.get(child_fieldname) or []) if row.name not in delete_set], + ) + existing = {c.name: c for c in (parent_doc.get(child_fieldname) or [])} + + # Apply updates + for upd in updates: + child_name = upd.get("name") + fields_dict = upd.get("fields") or {} + if not child_name or not fields_dict: + continue + if child_name not in existing: + frappe.throw(_("Child row '{0}' not found.").format(child_name)) + for fn, val in fields_dict.items(): + existing[child_name].set(fn, val) + + # Apply inserts + new_rows: list = [] + for ins in inserts: + fields_dict = ins.get("fields") or {} + new_row = parent_doc.append(child_fieldname, fields_dict) + new_rows.append((ins.get("temp_name", ""), new_row)) + + parent_doc.save() + + # For submitted docs (docstatus=1), Frappe's update_after_submit flow runs + # on_update_after_submit() AFTER db_update(). Any fields recalculated there + # (e.g. allocated_amount via calculate_contribution()) are updated in-memory + # but never flushed to DB. Force-persist them now. + if parent_doc.docstatus == 1: + for row in parent_doc.get(child_fieldname) or []: + row.db_update() + + frappe.db.commit() + + return { + "ok": True, + "inserted": [ + {"temp_name": temp_name, "name": row.name, "idx": row.idx} for temp_name, row in new_rows + ], + } + + except Exception as exc: + ev_err = _make_ev_error(exc) + frappe.message_log.clear() + frappe.db.rollback() + return ev_err # ── Child Table Data ───────────────────────────────────────────────────────── + @frappe.whitelist() def get_child_data(doctype: str, requests: str, parent_names: str) -> dict: """ @@ -639,7 +655,7 @@ def get_child_data(doctype: str, requests: str, parent_names: str) -> dict: children = frappe.get_all( child_doctype, filters={"parent": ["in", names], "parentfield": table_fn}, - fields=["parent"] + safe_fields, + fields=["parent", *safe_fields], order_by="idx asc", ignore_permissions=False, limit=0, @@ -671,377 +687,469 @@ def get_child_data(doctype: str, requests: str, parent_names: str) -> dict: @frappe.whitelist() def frappe_get(doctype: str, name: str, fieldname: str) -> dict: - """ - Fetch a single field value from one document. - - Three access patterns are supported: - - 1. Plain field: - FRAPPE_GET("Customer", "Tata Motors", "credit_limit") - - 2. Link-field traversal (dot notation — walks Link hops): - FRAPPE_GET("Sales Person", "Arjun Sharma", "employee.ctc") - → reads Sales Person.employee (Link→Employee), then Employee.ctc - - 3. Child-table row access (bracket notation): - FRAPPE_GET("Sales Invoice", "SINV-0001", "items[1].amount") - → returns the `amount` field of the 1st row in the items child table - The row index is 1-based. Returns None if row does not exist. - - Returns: - {"value": } - """ - import re - frappe.has_permission(doctype, "read", throw=True) - - # ── Pattern 3: child table bracket notation — "items[2].rate" ──────────── - child_match = re.match(r"^(\w+)\[(\d+)\]\.(\w+)$", fieldname) - if child_match: - child_field = child_match.group(1) - row_idx = int(child_match.group(2)) - 1 # 1-based → 0-based - child_fieldname = child_match.group(3) - - meta = frappe.get_meta(doctype) - df = meta.get_field(child_field) - if not df or df.fieldtype not in ("Table", "Table MultiSelect"): - frappe.throw(_(f"'{child_field}' is not a Table field on {doctype}")) - - doc = frappe.get_doc(doctype, name) - rows = doc.get(child_field) or [] - if row_idx < 0 or row_idx >= len(rows): - return {"value": None} - return {"value": rows[row_idx].get(child_fieldname)} - - # ── Pattern 2: Link-field traversal — "employee.ctc" ───────────────────── - if "." in fieldname: - parts = fieldname.split(".") - current_doctype = doctype - current_name = name - - for part in parts[:-1]: # all but last are link fields to traverse - _validate_fieldname(current_doctype, part) - meta = frappe.get_meta(current_doctype) - df = meta.get_field(part) - if not df or df.fieldtype != "Link": - frappe.throw(_(f"'{part}' is not a Link field on {current_doctype}")) - current_name = frappe.db.get_value(current_doctype, current_name, part) - if not current_name: - return {"value": None} - current_doctype = df.options - frappe.has_permission(current_doctype, "read", throw=True) - - final_field = parts[-1] - _validate_fieldname(current_doctype, final_field) - return {"value": frappe.db.get_value(current_doctype, current_name, final_field)} - - # ── Pattern 1: plain field ──────────────────────────────────────────────── - _validate_fieldname(doctype, fieldname) - return {"value": frappe.db.get_value(doctype, name, fieldname)} + """ + Fetch a single field value from one document. + + Three access patterns are supported: + + 1. Plain field: + FRAPPE_GET("Customer", "Tata Motors", "credit_limit") + + 2. Link-field traversal (dot notation — walks Link hops): + FRAPPE_GET("Sales Person", "Arjun Sharma", "employee.ctc") + → reads Sales Person.employee (Link→Employee), then Employee.ctc + + 3. Child-table row access (bracket notation): + FRAPPE_GET("Sales Invoice", "SINV-0001", "items[1].amount") + → returns the `amount` field of the 1st row in the items child table + The row index is 1-based. Returns None if row does not exist. + + Returns: + {"value": } + """ + import re + + frappe.has_permission(doctype, "read", throw=True) + + # ── Pattern 3: child table bracket notation — "items[2].rate" ──────────── + child_match = re.match(r"^(\w+)\[(\d+)\]\.(\w+)$", fieldname) + if child_match: + child_field = child_match.group(1) + row_idx = int(child_match.group(2)) - 1 # 1-based → 0-based + child_fieldname = child_match.group(3) + + meta = frappe.get_meta(doctype) + df = meta.get_field(child_field) + if not df or df.fieldtype not in ("Table", "Table MultiSelect"): + frappe.throw(_(f"'{child_field}' is not a Table field on {doctype}")) + + doc = frappe.get_doc(doctype, name) + rows = doc.get(child_field) or [] + if row_idx < 0 or row_idx >= len(rows): + return {"value": None} + return {"value": rows[row_idx].get(child_fieldname)} + + # ── Pattern 2: Link-field traversal — "employee.ctc" ───────────────────── + if "." in fieldname: + parts = fieldname.split(".") + current_doctype = doctype + current_name = name + + for part in parts[:-1]: # all but last are link fields to traverse + _validate_fieldname(current_doctype, part) + meta = frappe.get_meta(current_doctype) + df = meta.get_field(part) + if not df or df.fieldtype != "Link": + frappe.throw(_(f"'{part}' is not a Link field on {current_doctype}")) + current_name = frappe.db.get_value(current_doctype, current_name, part) + if not current_name: + return {"value": None} + current_doctype = df.options + frappe.has_permission(current_doctype, "read", throw=True) + + final_field = parts[-1] + _validate_fieldname(current_doctype, final_field) + return {"value": frappe.db.get_value(current_doctype, current_name, final_field)} + + # ── Pattern 1: plain field ──────────────────────────────────────────────── + _validate_fieldname(doctype, fieldname) + return {"value": frappe.db.get_value(doctype, name, fieldname)} + + +# ── Grid save helpers ───────────────────────────────────────────────────────── +# +# Frappe's own link check (Document._validate_links) has a gap: when a Link field +# has "fetch from" fields (for example Student Applicant.program fills class_name), +# get_invalid_links() reads the target with as_dict=True, gets None for a missing +# record, and skips the "invalid link" entry. A non-existent value is then saved. +# The grid needs one check that works for every Link field of every DocType. + +# Field types that carry no value of their own and must never be written by the grid. +_NON_VALUE_FIELDTYPES = frozenset( + {"Section Break", "Column Break", "Tab Break", "HTML", "Button", "Heading", "Fold"} +) +# Table fields are edited through the child-table panel, not through bulk_set_value. +_TABLE_FIELDTYPES = frozenset({"Table", "Table MultiSelect"}) +# Standard columns the grid may not change through bulk_set_value. +_PROTECTED_FIELDS = frozenset( + {"name", "owner", "creation", "modified", "modified_by", "docstatus", "idx", "doctype"} +) + + +def _check_writable_fields(meta, fields: dict) -> None: + """Refuse a field the grid is not allowed to write. + + The grid already blocks these in the browser. This is the same rule on the + server, so a direct call to bulk_set_value cannot bypass it. + """ + for fieldname in fields: + if fieldname in _PROTECTED_FIELDS: + frappe.throw(_("{0} cannot be changed here.").format(frappe.bold(fieldname))) + df = meta.get_field(fieldname) + if not df: + frappe.throw(_("{0} is not a field of {1}.").format(frappe.bold(fieldname), meta.name)) + if df.fieldtype in _NON_VALUE_FIELDTYPES or df.fieldtype in _TABLE_FIELDTYPES: + frappe.throw(_("{0} cannot be changed here.").format(frappe.bold(df.label or fieldname))) + # Read-only fields are never writable here, including fields filled by "fetch from": + # Frappe refreshes those itself on save, and the grid shows them locked. + if df.read_only or df.fieldtype == "Read Only": + frappe.throw( + _("{0} is read-only and cannot be changed here.").format(frappe.bold(df.label or fieldname)) + ) + + +def _check_link_values(doc, fieldnames=None) -> None: + """Refuse a Link value that does not exist, for any Link field of any DocType. + + Frappe's own check misses Link fields that have "fetch from" fields (see above), + so the values are checked directly. Only fields in `fieldnames` are checked when + it is given, so old wrong values on other fields do not block an unrelated edit. + """ + for df in doc.meta.get_link_fields(): + if fieldnames is not None and df.fieldname not in fieldnames: + continue + value = doc.get(df.fieldname) + target = df.options + if not value or not target: + continue + target_meta = frappe.get_meta(target) + if target_meta.get("is_virtual") or getattr(target_meta, "issingle", 0): + continue + if not frappe.db.exists(target, value): + frappe.throw( + _("{0}: {1} does not exist in {2}").format( + _(df.label or df.fieldname), frappe.bold(value), _(target) + ), + frappe.LinkValidationError, + ) @frappe.whitelist() def bulk_set_value(doctype: str, updates: str) -> dict: - """ - Update multiple existing records via the full Frappe ORM (doc.save()). - - Every write goes through validate, before_save, after_save, on_change and - all controller methods — identical to saving from the Frappe form UI. - doc.save() calls notify_update() internally, so realtime list_update events - are queued automatically and fired on the final commit. - - Submitted documents are rejected with a clear error; use the amend workflow. - - updates: JSON array of {name: str, fields: {fieldname: value, ...}} - Returns {"errors": [{"name": ..., "error": ...}, ...]} — empty list = all OK. - """ - frappe.has_permission(doctype, "write", throw=True) - - updates_data: list[dict] = (_fast_parse(updates) if isinstance(updates, str) else (updates or [])) - errors: list[dict] = [] - any_saved = False - - for item in updates_data: - name = item.get("name") - fields = item.get("fields") or {} - if not name or not fields: - continue - try: - doc = frappe.get_doc(doctype, name) - for fieldname, value in fields.items(): - doc.set(fieldname, value) - # doc.save() enforces all Frappe rules natively: - # - submitted docs: only allow_on_submit fields may be changed - # - cancelled docs: no edits allowed - # - validate, before_save, after_save hooks all run - doc.save() - any_saved = True - except Exception as exc: - errors.append({"name": name, "error": str(exc)}) - # frappe.throw() logs the message to frappe.message_log before - # raising, so it would appear in _server_messages and trigger - # Frappe's default msgprint alongside our EVModal. Clear it so - # the error is shown only through our custom UI. - frappe.message_log.clear() - - if any_saved: - # Explicit commit flushes all after_commit=True realtime events - # queued by doc.save() → notify_update() in the loop above. - frappe.db.commit() - - return {"errors": errors} + """ + Update multiple existing records via the full Frappe ORM (doc.save()). + + Every write goes through validate, before_save, after_save, on_change and + all controller methods — identical to saving from the Frappe form UI. + doc.save() calls notify_update() internally, so realtime list_update events + are queued automatically and fired on the final commit. + + Submitted documents are rejected with a clear error; use the amend workflow. + + Before the save, each row is checked for (a) fields the grid may not write + (read-only, system, layout and table fields) and (b) Link values that do not + exist. A row that fails is reported by name and the other rows still save. + + updates: JSON array of {name: str, fields: {fieldname: value, ...}} + Returns {"errors": [{"name": ..., "error": ..., "fields": [...]}, ...]} + — empty list = all OK. "fields" lists the fieldnames sent for that row, so the + grid can mark the failed cells. + """ + frappe.has_permission(doctype, "write", throw=True) + + updates_data: list[dict] = _fast_parse(updates) if isinstance(updates, str) else (updates or []) + meta = frappe.get_meta(doctype) + errors: list[dict] = [] + any_saved = False + + for item in updates_data: + name = item.get("name") + fields = item.get("fields") or {} + if not name or not fields: + continue + # One savepoint per row: if a hook fails half way, only this row is undone. + savepoint = "ev_bulk_row" + frappe.db.savepoint(savepoint) + try: + _check_writable_fields(meta, fields) + doc = frappe.get_doc(doctype, name) + for fieldname, value in fields.items(): + doc.set(fieldname, value) + _check_link_values(doc, set(fields)) + # doc.save() enforces all Frappe rules natively: + # - submitted docs: only allow_on_submit fields may be changed + # - cancelled docs: no edits allowed + # - validate, before_save, after_save hooks all run + doc.save() + any_saved = True + except Exception as exc: + frappe.db.rollback(save_point=savepoint) + errors.append({"name": name, "error": str(exc), "fields": list(fields)}) + # frappe.throw() logs the message to frappe.message_log before + # raising, so it would appear in _server_messages and trigger + # Frappe's default msgprint alongside our EVModal. Clear it so + # the error is shown only through our custom UI. + frappe.message_log.clear() + + if any_saved: + # Explicit commit flushes all after_commit=True realtime events + # queued by doc.save() → notify_update() in the loop above. + frappe.db.commit() + + return {"errors": errors} @frappe.whitelist() def bulk_create_records(doctype: str, rows: str) -> dict: - """ - Create multiple new Frappe documents from a JSON array of field-value dicts. - - Args: - doctype: Target DocType, e.g. "Customer" - rows: JSON string — [{fieldname: value, ...}, ...] - Empty-string values are skipped (treated as unset). - - Returns: - { - "created": [{"idx": 0, "name": "CUST-001"}, ...], - "errors": [{"idx": 1, "message": "Customer Name is required"}, ...] - } - - Security: caller must have Create permission on the DocType. - Each document is saved with the caller's permissions (no ignore_permissions). - """ - frappe.has_permission(doctype, "create", throw=True) - - rows_data: list[dict] = (_fast_parse(rows) if isinstance(rows, str) else (rows or [])) - created: list[dict] = [] - errors: list[dict] = [] - - for idx, row in enumerate(rows_data): - try: - doc = frappe.new_doc(doctype) - for field, value in row.items(): - # Skip empty values — let Frappe apply its own defaults - if value is not None and value != "": - doc.set(field, value) - doc.insert() - created.append({"idx": idx, "name": doc.name}) - except Exception as exc: - errors.append({"idx": idx, "message": str(exc)}) - - if created: - frappe.db.commit() - - return {"created": created, "errors": errors} + """ + Create multiple new Frappe documents from a JSON array of field-value dicts. + + Args: + doctype: Target DocType, e.g. "Customer" + rows: JSON string — [{fieldname: value, ...}, ...] + Empty-string values are skipped (treated as unset). + + Returns: + { + "created": [{"idx": 0, "name": "CUST-001"}, ...], + "errors": [{"idx": 1, "message": "Customer Name is required"}, ...] + } + + Security: caller must have Create permission on the DocType. + Each document is saved with the caller's permissions (no ignore_permissions). + """ + frappe.has_permission(doctype, "create", throw=True) + + rows_data: list[dict] = _fast_parse(rows) if isinstance(rows, str) else (rows or []) + created: list[dict] = [] + errors: list[dict] = [] + + for idx, row in enumerate(rows_data): + savepoint = "ev_bulk_create_row" + frappe.db.savepoint(savepoint) + try: + doc = frappe.new_doc(doctype) + for field, value in row.items(): + # Skip empty values — let Frappe apply its own defaults + if value is not None and value != "": + doc.set(field, value) + _check_link_values(doc) + doc.insert() + created.append({"idx": idx, "name": doc.name}) + except Exception as exc: + frappe.db.rollback(save_point=savepoint) + errors.append({"idx": idx, "message": str(exc)}) + + if created: + frappe.db.commit() + + return {"created": created, "errors": errors} @frappe.whitelist() def batch_check_links(doctype: str, rows_json: str, link_fields_json: str) -> dict: - """ - Check which linked values in the rows don't exist in their target DocTypes. - - Args: - doctype: Source DocType being imported into (for permission check). - rows_json: JSON array of {fieldname: value} dicts (already mapped). - link_fields_json: JSON array of {fieldname, options} describing Link fields. - - Returns: - {"missing": {fieldname: [val1, val2, ...]}} - Only fieldnames with at least one missing value are included. - """ - frappe.has_permission(doctype, "create", throw=True) - - rows: list[dict] = (_fast_parse(rows_json) if isinstance(rows_json, str) else (rows_json or [])) - link_fields: list[dict] = (_fast_parse(link_fields_json) if isinstance(link_fields_json, str) else (link_fields_json or [])) - - missing: dict[str, list[str]] = {} - - for lf in link_fields: - fn = lf.get("fieldname") - options_dt = lf.get("options") - if not fn or not options_dt: - continue - - # Collect unique non-empty values from the rows - values = list({str(r.get(fn, "")).strip() for r in rows if r.get(fn)}) - if not values: - continue - - try: - existing = { - r["name"] - for r in frappe.get_list( - options_dt, - filters=[["name", "in", values]], - fields=["name"], - limit=len(values) + 1, - ignore_permissions=False, - ) - } - except Exception: - continue - - missing_vals = [v for v in values if v not in existing] - if missing_vals: - missing[fn] = missing_vals - - return {"missing": missing} + """ + Check which linked values in the rows don't exist in their target DocTypes. + + Args: + doctype: Source DocType being imported into (for permission check). + rows_json: JSON array of {fieldname: value} dicts (already mapped). + link_fields_json: JSON array of {fieldname, options} describing Link fields. + + Returns: + {"missing": {fieldname: [val1, val2, ...]}} + Only fieldnames with at least one missing value are included. + """ + frappe.has_permission(doctype, "create", throw=True) + + rows: list[dict] = _fast_parse(rows_json) if isinstance(rows_json, str) else (rows_json or []) + link_fields: list[dict] = ( + _fast_parse(link_fields_json) if isinstance(link_fields_json, str) else (link_fields_json or []) + ) + + missing: dict[str, list[str]] = {} + + for lf in link_fields: + fn = lf.get("fieldname") + options_dt = lf.get("options") + if not fn or not options_dt: + continue + + # Collect unique non-empty values from the rows + values = list({str(r.get(fn, "")).strip() for r in rows if r.get(fn)}) + if not values: + continue + + try: + existing = { + r["name"] + for r in frappe.get_list( + options_dt, + filters=[["name", "in", values]], + fields=["name"], + limit=len(values) + 1, + ignore_permissions=False, + ) + } + except Exception: + continue + + missing_vals = [v for v in values if v not in existing] + if missing_vals: + missing[fn] = missing_vals + + return {"missing": missing} @frappe.whitelist() def create_link_record(doctype: str, value: str) -> dict: - """ - Create a minimal record in a Link DocType with the given name/value. - Determines the correct primary field from meta.autoname. - - Returns {"name": created_doc_name, "doctype": doctype} - """ - frappe.has_permission(doctype, "create", throw=True) - - meta = frappe.get_meta(doctype) - doc = frappe.new_doc(doctype) - autoname = meta.autoname or "" - - if autoname.lower().startswith("field:"): - # e.g. "field:item_code" → set that field - primary_field = autoname.split(":", 1)[1].strip() - doc.set(primary_field, value) - elif autoname.lower() in ("prompt", "name"): - doc.name = value - else: - # Fallback: set first required Data field that is empty - for f in meta.fields: - if f.reqd and f.fieldtype == "Data" and not doc.get(f.fieldname): - doc.set(f.fieldname, value) - break - - # Fill remaining required fields so insert() doesn't throw MandatoryError. - # - Data/Text fields → use `value` as a sensible default - # - Link fields → use first existing record in the linked doctype - # - Select fields → use first option in the options list - for f in meta.fields: - if not f.reqd or doc.get(f.fieldname): - continue - if f.fieldtype in ("Data", "Small Text", "Text", "Long Text"): - doc.set(f.fieldname, value) - elif f.fieldtype == "Link" and f.options: - first_val = frappe.db.get_value(f.options, {}, "name") - if first_val: - doc.set(f.fieldname, first_val) - elif f.fieldtype == "Select" and f.options: - first_opt = (f.options or "").strip().split("\n")[0] - if first_opt: - doc.set(f.fieldname, first_opt) - - doc.insert(ignore_permissions=False) - frappe.db.commit() - return {"name": doc.name, "doctype": doctype} + """ + Create a minimal record in a Link DocType with the given name/value. + Determines the correct primary field from meta.autoname. + + Returns {"name": created_doc_name, "doctype": doctype} + """ + frappe.has_permission(doctype, "create", throw=True) + + meta = frappe.get_meta(doctype) + doc = frappe.new_doc(doctype) + autoname = meta.autoname or "" + + if autoname.lower().startswith("field:"): + # e.g. "field:item_code" → set that field + primary_field = autoname.split(":", 1)[1].strip() + doc.set(primary_field, value) + elif autoname.lower() in ("prompt", "name"): + doc.name = value + else: + # Fallback: set first required Data field that is empty + for f in meta.fields: + if f.reqd and f.fieldtype == "Data" and not doc.get(f.fieldname): + doc.set(f.fieldname, value) + break + + # Fill remaining required fields so insert() doesn't throw MandatoryError. + # - Data/Text fields → use `value` as a sensible default + # - Link fields → use first existing record in the linked doctype + # - Select fields → use first option in the options list + for f in meta.fields: + if not f.reqd or doc.get(f.fieldname): + continue + if f.fieldtype in ("Data", "Small Text", "Text", "Long Text"): + doc.set(f.fieldname, value) + elif f.fieldtype == "Link" and f.options: + first_val = frappe.db.get_value(f.options, {}, "name") + if first_val: + doc.set(f.fieldname, first_val) + elif f.fieldtype == "Select" and f.options: + first_opt = (f.options or "").strip().split("\n")[0] + if first_opt: + doc.set(f.fieldname, first_opt) + + doc.insert(ignore_permissions=False) + frappe.db.commit() + return {"name": doc.name, "doctype": doctype} @frappe.whitelist() def frappe_child_get( - parent_doctype: str, - parent_name: str, - child_field: str, - row_index: int, - fieldname: str, + parent_doctype: str, + parent_name: str, + child_field: str, + row_index: int, + fieldname: str, ) -> dict: - """ - Fetch one field from a specific row of a child table. + """ + Fetch one field from a specific row of a child table. - Args: - parent_doctype: e.g. "Sales Invoice" - parent_name: e.g. "SINV-0001" - child_field: Table fieldname on the parent, e.g. "items" - row_index: 1-based row number (1 = first row) - fieldname: field to read from that child row, e.g. "amount" + Args: + parent_doctype: e.g. "Sales Invoice" + parent_name: e.g. "SINV-0001" + child_field: Table fieldname on the parent, e.g. "items" + row_index: 1-based row number (1 = first row) + fieldname: field to read from that child row, e.g. "amount" - Returns: - {"value": } — None if row does not exist + Returns: + {"value": } — None if row does not exist - Example: - FRAPPE_CHILD_GET("Sales Invoice", "SINV-0001", "items", 1, "amount") - → grand total of the first line item - """ - frappe.has_permission(parent_doctype, "read", throw=True) + Example: + FRAPPE_CHILD_GET("Sales Invoice", "SINV-0001", "items", 1, "amount") + → grand total of the first line item + """ + frappe.has_permission(parent_doctype, "read", throw=True) - meta = frappe.get_meta(parent_doctype) - df = meta.get_field(child_field) - if not df or df.fieldtype not in ("Table", "Table MultiSelect"): - frappe.throw(_(f"'{child_field}' is not a Table field on {parent_doctype}")) + meta = frappe.get_meta(parent_doctype) + df = meta.get_field(child_field) + if not df or df.fieldtype not in ("Table", "Table MultiSelect"): + frappe.throw(_(f"'{child_field}' is not a Table field on {parent_doctype}")) - row_idx = int(row_index) - 1 # 1-based → 0-based - if row_idx < 0: - frappe.throw(_("row_index must be ≥ 1")) + row_idx = int(row_index) - 1 # 1-based → 0-based + if row_idx < 0: + frappe.throw(_("row_index must be ≥ 1")) - doc = frappe.get_doc(parent_doctype, parent_name) - rows = doc.get(child_field) or [] - if row_idx >= len(rows): - return {"value": None} + doc = frappe.get_doc(parent_doctype, parent_name) + rows = doc.get(child_field) or [] + if row_idx >= len(rows): + return {"value": None} - return {"value": rows[row_idx].get(fieldname)} + return {"value": rows[row_idx].get(fieldname)} @frappe.whitelist() def frappe_aggregate( - doctype: str, - fieldname: str | None = None, - aggr_type: str = "sum", - fk1=None, fv1=None, - fk2=None, fv2=None, - fk3=None, fv3=None, - fk4=None, fv4=None, + doctype: str, + fieldname: str | None = None, + aggr_type: str = "sum", + fk1=None, + fv1=None, + fk2=None, + fv2=None, + fk3=None, + fv3=None, + fk4=None, + fv4=None, ) -> dict: - """ - Compute SUM, COUNT, or AVG over a DocType filtered by SUMIF-style pairs. + """ + Compute SUM, COUNT, or AVG over a DocType filtered by SUMIF-style pairs. - Accepts up to four field/value filter pairs. Operator-style keys are - supported: ``"transaction_date >="`` → ["transaction_date", ">=", value]. + Accepts up to four field/value filter pairs. Operator-style keys are + supported: ``"transaction_date >="`` → ["transaction_date", ">=", value]. - Returns: - {"value": } - """ - frappe.has_permission(doctype, "read", throw=True) + Returns: + {"value": } + """ + frappe.has_permission(doctype, "read", throw=True) - aggr_type = (aggr_type or "sum").lower() - if aggr_type not in ("sum", "count", "avg", "max", "min"): - frappe.throw(_("aggr_type must be 'sum', 'count', 'avg', 'max', or 'min'.")) + aggr_type = (aggr_type or "sum").lower() + if aggr_type not in ("sum", "count", "avg", "max", "min"): + frappe.throw(_("aggr_type must be 'sum', 'count', 'avg', 'max', or 'min'.")) - if aggr_type != "count" and not fieldname: - frappe.throw(_("fieldname is required for sum/avg/max/min aggregates.")) + if aggr_type != "count" and not fieldname: + frappe.throw(_("fieldname is required for sum/avg/max/min aggregates.")) - if aggr_type != "count" and fieldname: - _validate_fieldname(doctype, fieldname) + if aggr_type != "count" and fieldname: + _validate_fieldname(doctype, fieldname) - filters = _parse_filter_pairs(fk1, fv1, fk2, fv2, fk3, fv3, fk4, fv4) + filters = _parse_filter_pairs(fk1, fv1, fk2, fv2, fk3, fv3, fk4, fv4) - if aggr_type == "count": - return {"value": frappe.db.count(doctype, filters)} + if aggr_type == "count": + return {"value": frappe.db.count(doctype, filters)} - rows = frappe.get_list( - doctype, - filters=filters, - fields=[fieldname], - limit=_AGGREGATE_ROW_CAP, - ignore_permissions=False, - ) + rows = frappe.get_list( + doctype, + filters=filters, + fields=[fieldname], + limit=_AGGREGATE_ROW_CAP, + ignore_permissions=False, + ) - raw_vals = [r[fieldname] for r in rows if r.get(fieldname) is not None] + raw_vals = [r[fieldname] for r in rows if r.get(fieldname) is not None] - if not raw_vals: - return {"value": 0 if aggr_type in ("sum", "avg") else ""} + if not raw_vals: + return {"value": 0 if aggr_type in ("sum", "avg") else ""} - # max/min work on strings (ISO dates sort lexicographically) — no float cast - if aggr_type == "max": - return {"value": max(raw_vals)} - if aggr_type == "min": - return {"value": min(raw_vals)} + # max/min work on strings (ISO dates sort lexicographically) — no float cast + if aggr_type == "max": + return {"value": max(raw_vals)} + if aggr_type == "min": + return {"value": min(raw_vals)} - vals = [float(v) for v in raw_vals] + vals = [float(v) for v in raw_vals] - if aggr_type == "sum": - return {"value": sum(vals)} + if aggr_type == "sum": + return {"value": sum(vals)} - return {"value": sum(vals) / len(vals)} + return {"value": sum(vals) / len(vals)} _AGG_BATCH_TTL = 30 # seconds — formula cells feel live; Redis not hammered on every keystroke @@ -1050,6 +1158,7 @@ def frappe_aggregate( def _agg_batch_cache_key(raw: str) -> str: """Stable Redis key: user + MD5 of the raw JSON queries string.""" import hashlib + digest = hashlib.md5(raw.encode(), usedforsecurity=False).hexdigest() return f"ev_agg_batch:{frappe.session.user}:{digest}" @@ -1070,101 +1179,97 @@ def _invalidate_agg_cache_for_doctype(doc, method=None) -> None: def _exec_pregroup(q: dict) -> list: - """ - Execute a pre-grouped aggregate query (client sent fv1_list format) as a - single SQL IN-query. Returns a list of values parallel to q["fv1_list"]. - - This is called when the client has already done the grouping step and sent - {aggr_type, doctype, fieldname, fk1, fv1_list:[...], fk2-fk4 static filters}. - The server runs one GROUP BY query and maps results back to the input order. - """ - aggr_type = (q.get("aggr_type") or "sum").lower() - doctype = str(q.get("doctype") or "") - fieldname = str(q.get("fieldname") or "name") - fk1 = str(q.get("fk1") or "") - fv1_list = [str(v) for v in (q.get("fv1_list") or [])] - - if not doctype or not fk1 or not fv1_list: - return [] - - try: - frappe.has_permission(doctype, "read", throw=True) - except frappe.PermissionError: - return ["#PERM_DENIED"] * len(fv1_list) - - try: - if aggr_type not in ("count", "get"): - _validate_fieldname(doctype, fieldname) - if aggr_type != "get": - _validate_fieldname(doctype, fk1) - else: - _validate_fieldname(doctype, fieldname) - except Exception: - return ["#ARG!"] * len(fv1_list) - - # ── Static tail-filters (fk2-fk4) ──────────────────────────────────────── - extra_sql = "" - extra_params: list = [] - for fk, fv in ( - (q.get("fk2") or "", q.get("fv2")), - (q.get("fk3") or "", q.get("fv3")), - (q.get("fk4") or "", q.get("fv4")), - ): - if not fk or fv is None or fv == "": - continue - parts = fk.rsplit(" ", 1) - if len(parts) == 2 and parts[1].lower() in _FILTER_OPS: - field, op = parts[0].strip(), parts[1].strip() - else: - field, op = fk.strip(), "=" - try: - _validate_fieldname(doctype, field) - except Exception: - continue - extra_sql += f" AND `{field}` {op} %s" - extra_params.append(fv) - - placeholders = ", ".join(["%s"] * len(fv1_list)) - - # ── "get" path ──────────────────────────────────────────────────────────── - if aggr_type == "get": - sql = ( - f"SELECT `name`, `{fieldname}`" - f" FROM `tab{doctype}`" - f" WHERE `name` IN ({placeholders})" - ) - try: - rows = frappe.db.sql(sql, fv1_list, as_dict=False) - agg_map = {str(r[0]): r[1] for r in rows} - except Exception: - return [""] * len(fv1_list) - return [agg_map.get(fv1, "") for fv1 in fv1_list] - - # ── Aggregate path ──────────────────────────────────────────────────────── - if aggr_type == "sum": - agg_expr, default = f"SUM(`{fieldname}`)", 0 - elif aggr_type == "avg": - agg_expr, default = f"AVG(`{fieldname}`)", 0 - elif aggr_type == "max": - agg_expr, default = f"MAX(`{fieldname}`)", "" - elif aggr_type == "min": - agg_expr, default = f"MIN(`{fieldname}`)", "" - else: # count - agg_expr, default = "COUNT(*)", 0 - - sql = ( - f"SELECT `{fk1}`, {agg_expr}" - f" FROM `tab{doctype}`" - f" WHERE `{fk1}` IN ({placeholders}){extra_sql}" - f" GROUP BY `{fk1}`" - ) - try: - rows = frappe.db.sql(sql, fv1_list + extra_params, as_dict=False) - agg_map = {str(r[0]): r[1] for r in rows} - except Exception: - return [default] * len(fv1_list) - - return [agg_map.get(fv1, default) for fv1 in fv1_list] + """ + Execute a pre-grouped aggregate query (client sent fv1_list format) as a + single SQL IN-query. Returns a list of values parallel to q["fv1_list"]. + + This is called when the client has already done the grouping step and sent + {aggr_type, doctype, fieldname, fk1, fv1_list:[...], fk2-fk4 static filters}. + The server runs one GROUP BY query and maps results back to the input order. + """ + aggr_type = (q.get("aggr_type") or "sum").lower() + doctype = str(q.get("doctype") or "") + fieldname = str(q.get("fieldname") or "name") + fk1 = str(q.get("fk1") or "") + fv1_list = [str(v) for v in (q.get("fv1_list") or [])] + + if not doctype or not fk1 or not fv1_list: + return [] + + try: + frappe.has_permission(doctype, "read", throw=True) + except frappe.PermissionError: + return ["#PERM_DENIED"] * len(fv1_list) + + try: + if aggr_type not in ("count", "get"): + _validate_fieldname(doctype, fieldname) + if aggr_type != "get": + _validate_fieldname(doctype, fk1) + else: + _validate_fieldname(doctype, fieldname) + except Exception: + return ["#ARG!"] * len(fv1_list) + + # ── Static tail-filters (fk2-fk4) ──────────────────────────────────────── + extra_sql = "" + extra_params: list = [] + for fk, fv in ( + (q.get("fk2") or "", q.get("fv2")), + (q.get("fk3") or "", q.get("fv3")), + (q.get("fk4") or "", q.get("fv4")), + ): + if not fk or fv is None or fv == "": + continue + parts = fk.rsplit(" ", 1) + if len(parts) == 2 and parts[1].lower() in _FILTER_OPS: + field, op = parts[0].strip(), parts[1].strip() + else: + field, op = fk.strip(), "=" + try: + _validate_fieldname(doctype, field) + except Exception: + continue + extra_sql += f" AND `{field}` {op} %s" + extra_params.append(fv) + + placeholders = ", ".join(["%s"] * len(fv1_list)) + + # ── "get" path ──────────────────────────────────────────────────────────── + if aggr_type == "get": + sql = f"SELECT `name`, `{fieldname}`" f" FROM `tab{doctype}`" f" WHERE `name` IN ({placeholders})" + try: + rows = frappe.db.sql(sql, fv1_list, as_dict=False) + agg_map = {str(r[0]): r[1] for r in rows} + except Exception: + return [""] * len(fv1_list) + return [agg_map.get(fv1, "") for fv1 in fv1_list] + + # ── Aggregate path ──────────────────────────────────────────────────────── + if aggr_type == "sum": + agg_expr, default = f"SUM(`{fieldname}`)", 0 + elif aggr_type == "avg": + agg_expr, default = f"AVG(`{fieldname}`)", 0 + elif aggr_type == "max": + agg_expr, default = f"MAX(`{fieldname}`)", "" + elif aggr_type == "min": + agg_expr, default = f"MIN(`{fieldname}`)", "" + else: # count + agg_expr, default = "COUNT(*)", 0 + + sql = ( + f"SELECT `{fk1}`, {agg_expr}" + f" FROM `tab{doctype}`" + f" WHERE `{fk1}` IN ({placeholders}){extra_sql}" + f" GROUP BY `{fk1}`" + ) + try: + rows = frappe.db.sql(sql, fv1_list + extra_params, as_dict=False) + agg_map = {str(r[0]): r[1] for r in rows} + except Exception: + return [default] * len(fv1_list) + + return [agg_map.get(fv1, default) for fv1 in fv1_list] @frappe.whitelist() @@ -1184,12 +1289,17 @@ def frappe_aggregate_batch(queries: str) -> dict: {"results": [value, ...]} — parallel to the input *queries* array. """ raw: str = queries if isinstance(queries, str) else frappe.as_json(queries) - qs: list = (_fast_parse(raw) if isinstance(raw, str) else (raw or [])) if isinstance(queries, str) else (queries or []) + qs: list = ( + (_fast_parse(raw) if isinstance(raw, str) else (raw or [])) + if isinstance(queries, str) + else (queries or []) + ) if not qs: return {"results": []} # ── Redis cache check ──────────────────────────────────────────────────── import time as _time + t0 = _time.monotonic() cache_key = _agg_batch_cache_key(raw) cached = frappe.cache().get_value(cache_key) @@ -1203,8 +1313,8 @@ def frappe_aggregate_batch(queries: str) -> dict: # Group key: (aggr_type, doctype, fieldname, fk1 [simple field — no operator], # fk2+fv2, fk3+fv3, fk4+fv4 static filters) # Only fv1 varies within a group — each row has a different FK value. - groups: dict = {} # group_key → [(index, fv1), ...] - solo: list = [] # [(index, query_dict), ...] — cannot be batched + groups: dict = {} # group_key → [(index, fv1), ...] + solo: list = [] # [(index, query_dict), ...] — cannot be batched for i, q in enumerate(qs): # ── Pre-grouped compact format (client sent fv1_list) ───────────────── @@ -1216,7 +1326,7 @@ def frappe_aggregate_batch(queries: str) -> dict: continue aggr_type = (q.get("aggr_type") or "sum").lower() - doctype = str(q.get("doctype") or "") + doctype = str(q.get("doctype") or "") fieldname = str(q.get("fieldname") or "name") fk1 = str(q.get("fk1") or "") fv1 = q.get("fv1") @@ -1229,19 +1339,20 @@ def frappe_aggregate_batch(queries: str) -> dict: # Batch only when fk1 is a plain field name (no trailing operator like ">=" # or "!=") and fv1 is a non-empty scalar. - can_group = bool( - fk1 - and fv1 is not None - and fv1 != "" - and " " not in fk1.strip() - ) + can_group = bool(fk1 and fv1 is not None and fv1 != "" and " " not in fk1.strip()) if can_group: gk = ( - aggr_type, doctype, fieldname, fk1, - fk2, str(fv2) if fv2 is not None else "", - fk3, str(fv3) if fv3 is not None else "", - fk4, str(fv4) if fv4 is not None else "", + aggr_type, + doctype, + fieldname, + fk1, + fk2, + str(fv2) if fv2 is not None else "", + fk3, + str(fv3) if fv3 is not None else "", + fk4, + str(fv4) if fv4 is not None else "", ) groups.setdefault(gk, []).append((i, fv1)) else: @@ -1275,6 +1386,13 @@ def frappe_aggregate_batch(queries: str) -> dict: results[idx] = "#ARG!" continue + # Map fv1 → [indices] (handles duplicate FK values across rows). Built before the "get" + # branch below, which needs both of these. + fv1_to_idxs: dict = {} + for idx, fv1 in items: + fv1_to_idxs.setdefault(str(fv1), []).append(idx) + unique_vals = list(fv1_to_idxs.keys()) + # ── "get" path: SELECT name, fieldname FROM tab WHERE name IN (...) ── if aggr_type == "get": try: @@ -1284,12 +1402,22 @@ def frappe_aggregate_batch(queries: str) -> dict: f" FROM `tab{doctype}`" f" WHERE `name` IN ({placeholders})" ) - rows = frappe.db.sql(sql, unique_vals, as_dict=False) + rows = frappe.db.sql(sql, unique_vals, as_dict=False) agg_map = {str(r[0]): r[1] for r in rows} except Exception: for idx, fv1 in items: - solo.append((idx, {"aggr_type": "get", "doctype": doctype, - "fieldname": fieldname, "fk1": "name", "fv1": fv1})) + solo.append( + ( + idx, + { + "aggr_type": "get", + "doctype": doctype, + "fieldname": fieldname, + "fk1": "name", + "fv1": fv1, + }, + ) + ) continue for fv1_str, idxs in fv1_to_idxs.items(): val = agg_map.get(fv1_str, "") @@ -1297,20 +1425,20 @@ def frappe_aggregate_batch(queries: str) -> dict: results[idx] = val continue - # Map fv1 → [indices] (handles duplicate FK values across rows) - fv1_to_idxs: dict = {} - for idx, fv1 in items: - fv1_to_idxs.setdefault(str(fv1), []).append(idx) - unique_vals = list(fv1_to_idxs.keys()) - if len(unique_vals) == 1: # Single distinct value — delegate to the existing scalar endpoint res = frappe_aggregate( - doctype=doctype, fieldname=fieldname, aggr_type=aggr_type, - fk1=fk1, fv1=unique_vals[0], - fk2=fk2 or None, fv2=fv2s or None, - fk3=fk3 or None, fv3=fv3s or None, - fk4=fk4 or None, fv4=fv4s or None, + doctype=doctype, + fieldname=fieldname, + aggr_type=aggr_type, + fk1=fk1, + fv1=unique_vals[0], + fk2=fk2 or None, + fv2=fv2s or None, + fk3=fk3 or None, + fv3=fv3s or None, + fk4=fk4 or None, + fv4=fv4s or None, ) val = res.get("value", 0) for idx in fv1_to_idxs[unique_vals[0]]: @@ -1318,29 +1446,29 @@ def frappe_aggregate_batch(queries: str) -> dict: continue # Build IN-query SQL (fk1 and fieldname already validated — safe as identifiers) - table = f"`tab{doctype}`" + table = f"`tab{doctype}`" fk1_col = f"`{fk1}`" if aggr_type == "sum": agg_expr = f"SUM(`{fieldname}`)" - default = 0 + default = 0 elif aggr_type == "avg": agg_expr = f"AVG(`{fieldname}`)" - default = 0 + default = 0 elif aggr_type == "max": agg_expr = f"MAX(`{fieldname}`)" - default = "" + default = "" elif aggr_type == "min": agg_expr = f"MIN(`{fieldname}`)" - default = "" + default = "" else: # count agg_expr = "COUNT(*)" - default = 0 + default = 0 # Static tail-filters (fk2-fk4). # Supports operator-style keys like "transaction_date >=" — split off the # operator before validating the fieldname and before embedding in SQL. - extra_sql = "" + extra_sql = "" extra_params: list = [] for fk, fv in ((fk2, fv2s), (fk3, fv3s), (fk4, fv4s)): if not fk or not fv: @@ -1354,7 +1482,7 @@ def frappe_aggregate_batch(queries: str) -> dict: _validate_fieldname(doctype, field) except Exception: continue - extra_sql += f" AND `{field}` {op} %s" + extra_sql += f" AND `{field}` {op} %s" extra_params.append(fv) placeholders = ", ".join(["%s"] * len(unique_vals)) @@ -1367,16 +1495,29 @@ def frappe_aggregate_batch(queries: str) -> dict: params = unique_vals + extra_params try: - rows = frappe.db.sql(sql, params, as_dict=False) + rows = frappe.db.sql(sql, params, as_dict=False) agg_map = {str(r[0]): r[1] for r in rows} except Exception: # SQL failure — fall back to individual scalar calls for this group - for idx, q in [(idx, {"aggr_type": aggr_type, "doctype": doctype, - "fieldname": fieldname, "fk1": fk1, "fv1": fv1, - "fk2": fk2 or None, "fv2": fv2s or None, - "fk3": fk3 or None, "fv3": fv3s or None, - "fk4": fk4 or None, "fv4": fv4s or None}) - for idx, fv1 in items]: + for idx, q in [ + ( + idx, + { + "aggr_type": aggr_type, + "doctype": doctype, + "fieldname": fieldname, + "fk1": fk1, + "fv1": fv1, + "fk2": fk2 or None, + "fv2": fv2s or None, + "fk3": fk3 or None, + "fv3": fv3s or None, + "fk4": fk4 or None, + "fv4": fv4s or None, + }, + ) + for idx, fv1 in items + ]: solo.append((idx, q)) continue @@ -1385,945 +1526,1059 @@ def frappe_aggregate_batch(queries: str) -> dict: for idx in idxs: results[idx] = val - # ── Execute solo queries (ungroupable — operator filters, empty fv1, etc.) ─ - # (also receives "get" fallbacks pushed here by the group section) - for idx, q in solo: - try: - aggr_type_s = (q.get("aggr_type") or "sum").lower() - if aggr_type_s == "get": - # Single FRAPPE_GET fallback — use frappe.get_value - val = frappe.get_value( - q.get("doctype", ""), - q.get("fv1"), # the document name - q.get("fieldname") or "name", + # ── Execute solo queries (ungroupable — operator filters, empty fv1, etc.) ─ + # (also receives "get" fallbacks pushed here by the group section) + for idx, q in solo: + try: + aggr_type_s = (q.get("aggr_type") or "sum").lower() + if aggr_type_s == "get": + # Single FRAPPE_GET fallback — use frappe.get_value + val = frappe.get_value( + q.get("doctype", ""), + q.get("fv1"), # the document name + q.get("fieldname") or "name", + ) + results[idx] = val if val is not None else "" + else: + res = frappe_aggregate( + doctype=q.get("doctype", ""), + fieldname=q.get("fieldname") or "name", + aggr_type=aggr_type_s, + fk1=q.get("fk1"), + fv1=q.get("fv1"), + fk2=q.get("fk2"), + fv2=q.get("fv2"), + fk3=q.get("fk3"), + fv3=q.get("fv3"), + fk4=q.get("fk4"), + fv4=q.get("fv4"), + ) + results[idx] = res.get("value", 0) + except Exception: + results[idx] = "#ERR!" + + # ── Cache and return ───────────────────────────────────────────────────── + elapsed_ms = round((_time.monotonic() - t0) * 1000) + out = {"results": results, "_cache": "miss", "_ms": elapsed_ms} + frappe.cache().set_value(cache_key, out, expires_in_sec=_AGG_BATCH_TTL) + return out + + +# ── Dashboard formula-column card aggregate ──────────────────────────────────── + + +@frappe.whitelist() +def compute_card_aggregate( + doctype: str, + formula: str, + col_fieldnames: str, + filter_op: str, + filter_val: str, + aggregate: str, + agg_fieldname: str | None = None, + list_filters: str | None = None, + extra_filters: str | None = None, + period_start: str | None = None, + period_end: str | None = None, +) -> dict: + """ + Server-side number card computation for formula columns (⚡). + + Fetches ALL rows for the DocType (no page-limit cap), evaluates the + HyperFormula template formula for every row, applies the card filter and + any extra plain-field filters, then returns the requested aggregate. + + Parameters + ---------- + doctype : Source DocType (the sheet's doctype) + formula : HyperFormula formula template, e.g. + ``=IF(FRAPPE_COUNT("Sales Order","customer",A1)>0,"Active","Inactive")`` + col_fieldnames : JSON array of fieldnames in column order (A→[0], B→[1], …) + filter_op : Operator on the formula result: ``=``, ``!=``, ``contains``, + ``>``, ``<``, ``>=``, ``<=`` + filter_val : Value to compare the formula result against + aggregate : ``count`` | ``sum`` | ``avg`` | ``min`` | ``max`` + agg_fieldname : DB fieldname for sum/avg/min/max (ignored for count) + list_filters : JSON array of Frappe filter tuples (current sheet filters) + extra_filters : JSON array of ``{col, op, val}`` dicts for non-formula + card filters applied after formula evaluation + """ + import json + + _check_doctype_permission(doctype) + + col_fields: list = ( + _fast_parse(col_fieldnames) if isinstance(col_fieldnames, str) else (col_fieldnames or []) + ) + lst_filters: list = _fast_parse(list_filters) if isinstance(list_filters, str) and list_filters else [] + extra: list = _fast_parse(extra_filters) if isinstance(extra_filters, str) and extra_filters else [] + + # agg_fieldname may be a formula/virtual col key (starts with _) — never a real DB field + agg_field = (agg_fieldname or "").strip() or None + if agg_field and agg_field.startswith("_"): + agg_field = None + + # Substitute PERIOD_START() / PERIOD_END() placeholders with actual dates + # (the client resolves these via formula_manager; pass them down so the server + # can build correct SQL filters for formulas like FRAPPE_COUNT(...,PERIOD_START(),...)) + if period_start or period_end: + formula = formula.replace("PERIOD_START()", f'"{period_start or ""}"') + formula = formula.replace("PERIOD_END()", f'"{period_end or ""}"') + + # ── Determine which DB fields we need ──────────────────────────────── + import re + + fetch_fields: set = {"name"} + for ref in re.findall(r"([A-Z]+)1\b", formula, re.IGNORECASE): + idx = _col_letter_to_index(ref.upper()) + if idx < len(col_fields): + fetch_fields.add(col_fields[idx]) + # agg_field is None if it was a virtual/formula col — skip it + if agg_field: + fetch_fields.add(agg_field) + for ef in extra: + col = (ef.get("col") or "").strip() + if col and not col.startswith("_"): + fetch_fields.add(col) + + # ── Strategy 1: SQL-first (no row fetching) ─────────────────────────── + # Parses the formula into a subquery + COUNT — 2 DB queries, zero memory. + sql_result = _try_sql_formula_compute( + doctype, + formula, + col_fields, + filter_op, + str(filter_val), + aggregate, + agg_field, + lst_filters, + extra, + ) + if sql_result is not None: + return {"value": sql_result} + + # ── Strategy 2: Row-fetch fallback (unparseable formulas) ───────────── + # Hard cap: 50 000 rows. Warns if data was truncated. + _ROW_CAP = 50_000 + rows = frappe.get_all( + doctype, + fields=list(fetch_fields), + filters=lst_filters, + limit=_ROW_CAP + 1, + ) + truncated = len(rows) > _ROW_CAP + if truncated: + rows = rows[:_ROW_CAP] + + # ── Evaluate HyperFormula template for every row → row["_val"] ─────── + _eval_formula_bulk(formula, rows, col_fields) + + # ── Apply formula column filter ─────────────────────────────────────── + rows = [r for r in rows if _card_match(str(r.get("_val", "")), filter_op, str(filter_val))] + + # ── Apply any extra plain-field card filters ────────────────────────── + for ef in extra: + col, op, val = (ef.get("col") or ""), (ef.get("op") or "="), str(ef.get("val") or "") + if not col or col.startswith("_"): + continue + rows = [r for r in rows if _card_match(str(r.get(col, "") or ""), op, val)] + + # ── Compute aggregate ───────────────────────────────────────────────── + if aggregate == "count": + result = len(rows) + return {"value": result, "truncated": truncated} + + if not agg_field: + return {"value": len(rows), "truncated": truncated} + + nums = [] + for r in rows: + try: + nums.append(float(r.get(agg_field) or 0)) + except (TypeError, ValueError): + pass + + if not nums: + return {"value": 0} + + if aggregate == "sum": + return {"value": round(sum(nums), 2), "truncated": truncated} + if aggregate == "avg": + return {"value": round(sum(nums) / len(nums), 2), "truncated": truncated} + if aggregate == "min": + return {"value": round(min(nums), 2), "truncated": truncated} + if aggregate == "max": + return {"value": round(max(nums), 2), "truncated": truncated} + return {"value": len(rows), "truncated": truncated} + + +# ── SQL-first formula compute ────────────────────────────────────────────────── + + +def _try_sql_formula_compute( + doctype, + formula, + col_fields, + filter_op, + filter_val, + aggregate, + agg_field, + lst_filters, + extra_filters, +): + """ + Push formula-column card computation entirely into SQL — zero row fetching. + + Supports (any combination): + =IF(FRAPPE_COUNT("DT", [fk,fv …], A1, …) op N, "t", "f") + =IF(FRAPPE_SUM ("DT","sum_field",[fk,fv …],A1,…) op N, "t","f") + =IF(FRAPPE_AVG (…) op N, "t", "f") + =IF(FRAPPE_GET ("DT", A1, "field") op "val", "t", "f") + =IF(A1 op "val", "t", "f") ← direct column comparison + =IF(A1 op "val", "t", "f") with filter_op "!=" / "contains" / … + + Also handles filter_op other than "=" by inverting the want_true logic. + + Returns the computed value (int/float) or None (caller falls back to row-fetch). + """ + import re + + expr = formula.strip().lstrip("=").strip() + + # ── Require IF(condition, true_val, false_val) ──────────────────────── + parsed = _parse_if_expr(expr) + if not parsed: + return None + cond_str, true_val, false_val = parsed + + # Determine which branch the card filter selects + filter_val_s = str(filter_val).strip() + true_val_s = str(true_val).strip() + false_val_s = str(false_val).strip() + + if filter_op == "=": + if filter_val_s == true_val_s: + want_true = True + elif filter_val_s == false_val_s: + want_true = False + else: + return None + elif filter_op == "!=": + if filter_val_s == true_val_s: + want_true = False + elif filter_val_s == false_val_s: + want_true = True + else: + return None + else: + # contains / numeric ops on formula strings — can't push to SQL + return None + + # Collect plain (non-formula) extra card filters + plain_extra = [ + [ef["col"], ef.get("op", "="), ef.get("val", "")] + for ef in extra_filters + if (ef.get("col") or "").strip() and not ef["col"].startswith("_") + ] + + # ── Dispatch by condition type ──────────────────────────────────────── + return _sql_dispatch_condition( + cond_str, + want_true, + doctype, + col_fields, + aggregate, + agg_field, + lst_filters, + plain_extra, + ) + + +def _sql_dispatch_condition( + cond_str, want_true, doctype, col_fields, aggregate, agg_field, lst_filters, plain_extra +): + """Route a parsed IF-condition to the right SQL handler.""" + import re + + # ── FRAPPE_COUNT / FRAPPE_SUM / FRAPPE_AVG (...) op N ───────────────── + m_agg = re.match( + r"^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG)\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$", + cond_str, + re.I | re.S, + ) + if m_agg: + func = m_agg.group(1).upper() + args = _parse_formula_args(m_agg.group(2)) + comp_op = m_agg.group(3).replace("<>", "!=") + try: + threshold = float(m_agg.group(4).strip().strip('"')) + except (ValueError, TypeError): + return None + return _sql_frappe_numeric_func( + func, + args, + comp_op, + threshold, + want_true, + doctype, + col_fields, + aggregate, + agg_field, + lst_filters, + plain_extra, + ) + + # ── FRAPPE_GET("DT", A1, "field") op "val" ──────────────────────────── + m_get = re.match( + r"^FRAPPE_GET\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$", + cond_str, + re.I | re.S, + ) + if m_get: + args = _parse_formula_args(m_get.group(1)) + comp_op = m_get.group(2).replace("<>", "!=") + val = _strip_formula_quotes(m_get.group(3).strip()) + return _sql_frappe_get( + args, + comp_op, + val, + want_true, + doctype, + col_fields, + aggregate, + agg_field, + lst_filters, + plain_extra, + ) + + # ── A1 op "val" — direct column comparison ──────────────────────────── + m_col = re.match(r"^([A-Z]+)1\s*([><=!]+|<>)\s*(.+)$", cond_str, re.I) + if m_col: + idx = _col_letter_to_index(m_col.group(1).upper()) + row_field = col_fields[idx] if idx < len(col_fields) else "name" + comp_op = m_col.group(2).replace("<>", "!=") + val = _strip_formula_quotes(m_col.group(3).strip()) + neg_op = {"=": "!=", "!=": "=", ">": "<=", "<": ">=", ">=": "<", "<=": ">"} + op = comp_op if want_true else neg_op.get(comp_op, "!=") + try: + return _count_or_agg( + doctype, + lst_filters + plain_extra + [[row_field, op, val]], + aggregate, + agg_field, + ) + except Exception: + return None + + return None + + +def _sql_frappe_numeric_func( + func, + args, + comp_op, + threshold, + want_true, + doctype, + col_fields, + aggregate, + agg_field, + lst_filters, + plain_extra, +): + """ + Handle IF(FRAPPE_COUNT/SUM/AVG(…) op N, t, f). + Builds a GROUP BY query on the link DocType, resolves passing keys, + then counts/aggregates on the base DocType with IN / NOT IN. + """ + if not args: + return None + + link_doctype = _strip_formula_quotes(args[0]) + if not link_doctype: + return None + + sum_field, arg_start = None, 1 + if func in ("FRAPPE_SUM", "FRAPPE_AVG"): + if len(args) < 2: + return None + sum_field = _strip_formula_quotes(args[1]) + arg_start = 2 + + # Parse fk/fv pairs — exactly one must be a dynamic cell reference (A1) + static_link_filters: list = [] + dynamic_pair = None + + i = arg_start + while i + 1 < len(args): + fk = _strip_formula_quotes(args[i]) + fv_raw = args[i + 1].strip() + + col_ref = _try_col_ref(fv_raw) + if col_ref is not None: + if dynamic_pair is not None: + return None # multiple dynamic cols — can't vectorise + rf = col_fields[col_ref] if col_ref < len(col_fields) else "name" + dynamic_pair = (fk, rf) + else: + # Operator may be embedded in fk: "transaction_date >=" + import re + + fk_m = re.match(r"^(.+?)\s+(>=|<=|!=|<>|>|<|=|like)\s*$", fk, re.I) + if fk_m: + static_link_filters.append( + [fk_m.group(1).strip(), fk_m.group(2).strip(), _strip_formula_quotes(fv_raw)] ) - results[idx] = val if val is not None else "" else: - res = frappe_aggregate( - doctype = q.get("doctype", ""), - fieldname = q.get("fieldname") or "name", - aggr_type = aggr_type_s, - fk1=q.get("fk1"), fv1=q.get("fv1"), - fk2=q.get("fk2"), fv2=q.get("fv2"), - fk3=q.get("fk3"), fv3=q.get("fv3"), - fk4=q.get("fk4"), fv4=q.get("fv4"), - ) - results[idx] = res.get("value", 0) - except Exception: - results[idx] = "#ERR!" + static_link_filters.append([fk, "=", _strip_formula_quotes(fv_raw)]) + i += 2 - # ── Cache and return ───────────────────────────────────────────────────── - elapsed_ms = round((_time.monotonic() - t0) * 1000) - out = {"results": results, "_cache": "miss", "_ms": elapsed_ms} - frappe.cache().set_value(cache_key, out, expires_in_sec=_AGG_BATCH_TTL) - return out + if dynamic_pair is None: + return None + link_field, row_field = dynamic_pair -# ── Dashboard formula-column card aggregate ──────────────────────────────────── + # ── Query 1: GROUP BY → per-key numeric value ───────────────────────── + try: + if func == "FRAPPE_COUNT": + rows = frappe.db.get_all( + link_doctype, + fields=[link_field, "count(*) as _v"], + filters=static_link_filters, + group_by=link_field, + ) + val_map = {str(r.get(link_field, "")): (r.get("_v") or 0) for r in rows} + + elif func == "FRAPPE_SUM" and sum_field: + rows = frappe.db.get_all( + link_doctype, + fields=[link_field, f"sum({sum_field}) as _v"], + filters=static_link_filters, + group_by=link_field, + ) + val_map = {str(r.get(link_field, "")): float(r.get("_v") or 0) for r in rows} + + elif func == "FRAPPE_AVG" and sum_field: + rows = frappe.db.get_all( + link_doctype, + fields=[link_field, f"sum({sum_field}) as _s", "count(*) as _c"], + filters=static_link_filters, + group_by=link_field, + ) + val_map = { + str(r.get(link_field, "")): float(r.get("_s") or 0) / max(int(r.get("_c") or 1), 1) + for r in rows + } + else: + return None + except Exception: + return None + # ── Classify keys: passing / failing / absent ───────────────────────── + passing_keys = {k for k, v in val_map.items() if _num_compare(v, comp_op, threshold)} + failing_keys = {k for k in val_map if k not in passing_keys} + zero_passes = _num_compare(0, comp_op, threshold) -@frappe.whitelist() -def compute_card_aggregate( - doctype: str, - formula: str, - col_fieldnames: str, - filter_op: str, - filter_val: str, - aggregate: str, - agg_fieldname: str | None = None, - list_filters: str | None = None, - extra_filters: str | None = None, - period_start: str | None = None, - period_end: str | None = None, -) -> dict: - """ - Server-side number card computation for formula columns (⚡). - - Fetches ALL rows for the DocType (no page-limit cap), evaluates the - HyperFormula template formula for every row, applies the card filter and - any extra plain-field filters, then returns the requested aggregate. - - Parameters - ---------- - doctype : Source DocType (the sheet's doctype) - formula : HyperFormula formula template, e.g. - ``=IF(FRAPPE_COUNT("Sales Order","customer",A1)>0,"Active","Inactive")`` - col_fieldnames : JSON array of fieldnames in column order (A→[0], B→[1], …) - filter_op : Operator on the formula result: ``=``, ``!=``, ``contains``, - ``>``, ``<``, ``>=``, ``<=`` - filter_val : Value to compare the formula result against - aggregate : ``count`` | ``sum`` | ``avg`` | ``min`` | ``max`` - agg_fieldname : DB fieldname for sum/avg/min/max (ignored for count) - list_filters : JSON array of Frappe filter tuples (current sheet filters) - extra_filters : JSON array of ``{col, op, val}`` dicts for non-formula - card filters applied after formula evaluation - """ - import json - - _check_doctype_permission(doctype) - - col_fields: list = (_fast_parse(col_fieldnames) if isinstance(col_fieldnames, str) else (col_fieldnames or [])) - lst_filters: list = _fast_parse(list_filters) if isinstance(list_filters, str) and list_filters else [] - extra: list = _fast_parse(extra_filters) if isinstance(extra_filters, str) and extra_filters else [] - - # agg_fieldname may be a formula/virtual col key (starts with _) — never a real DB field - agg_field = (agg_fieldname or "").strip() or None - if agg_field and agg_field.startswith("_"): - agg_field = None - - # Substitute PERIOD_START() / PERIOD_END() placeholders with actual dates - # (the client resolves these via formula_manager; pass them down so the server - # can build correct SQL filters for formulas like FRAPPE_COUNT(...,PERIOD_START(),...)) - if period_start or period_end: - formula = formula.replace("PERIOD_START()", f'"{period_start or ""}"') - formula = formula.replace("PERIOD_END()", f'"{period_end or ""}"') - - # ── Determine which DB fields we need ──────────────────────────────── - import re - - fetch_fields: set = {"name"} - for ref in re.findall(r'([A-Z]+)1\b', formula, re.IGNORECASE): - idx = _col_letter_to_index(ref.upper()) - if idx < len(col_fields): - fetch_fields.add(col_fields[idx]) - # agg_field is None if it was a virtual/formula col — skip it - if agg_field: - fetch_fields.add(agg_field) - for ef in extra: - col = (ef.get("col") or "").strip() - if col and not col.startswith("_"): - fetch_fields.add(col) - - # ── Strategy 1: SQL-first (no row fetching) ─────────────────────────── - # Parses the formula into a subquery + COUNT — 2 DB queries, zero memory. - sql_result = _try_sql_formula_compute( - doctype, formula, col_fields, - filter_op, str(filter_val), aggregate, agg_field, - lst_filters, extra, - ) - if sql_result is not None: - return {"value": sql_result} - - # ── Strategy 2: Row-fetch fallback (unparseable formulas) ───────────── - # Hard cap: 50 000 rows. Warns if data was truncated. - _ROW_CAP = 50_000 - rows = frappe.get_all( - doctype, - fields=list(fetch_fields), - filters=lst_filters, - limit=_ROW_CAP + 1, - ) - truncated = len(rows) > _ROW_CAP - if truncated: - rows = rows[:_ROW_CAP] - - # ── Evaluate HyperFormula template for every row → row["_val"] ─────── - _eval_formula_bulk(formula, rows, col_fields) - - # ── Apply formula column filter ─────────────────────────────────────── - rows = [r for r in rows if _card_match(str(r.get("_val", "")), filter_op, str(filter_val))] - - # ── Apply any extra plain-field card filters ────────────────────────── - for ef in extra: - col, op, val = (ef.get("col") or ""), (ef.get("op") or "="), str(ef.get("val") or "") - if not col or col.startswith("_"): - continue - rows = [r for r in rows if _card_match(str(r.get(col, "") or ""), op, val)] - - # ── Compute aggregate ───────────────────────────────────────────────── - if aggregate == "count": - result = len(rows) - return {"value": result, "truncated": truncated} - - if not agg_field: - return {"value": len(rows), "truncated": truncated} - - nums = [] - for r in rows: - try: - nums.append(float(r.get(agg_field) or 0)) - except (TypeError, ValueError): - pass - - if not nums: - return {"value": 0} - - if aggregate == "sum": - return {"value": round(sum(nums), 2), "truncated": truncated} - if aggregate == "avg": - return {"value": round(sum(nums) / len(nums), 2), "truncated": truncated} - if aggregate == "min": - return {"value": round(min(nums), 2), "truncated": truncated} - if aggregate == "max": - return {"value": round(max(nums), 2), "truncated": truncated} - return {"value": len(rows), "truncated": truncated} + return _sql_apply_in_not_in( + doctype, + row_field, + passing_keys, + failing_keys, + zero_passes, + want_true, + lst_filters, + plain_extra, + aggregate, + agg_field, + ) -# ── SQL-first formula compute ────────────────────────────────────────────────── +def _sql_frappe_get( + args, comp_op, val, want_true, doctype, col_fields, aggregate, agg_field, lst_filters, plain_extra +): + """ + Handle IF(FRAPPE_GET("link_dt", A1, "field") op "val", t, f). + Fetches matching names from link_dt, then IN/NOT IN on base doctype. + """ + if len(args) < 3: + return None + link_doctype = _strip_formula_quotes(args[0]) + fv_raw = args[1].strip() + target_field = _strip_formula_quotes(args[2]) -def _try_sql_formula_compute( - doctype, formula, col_fields, - filter_op, filter_val, aggregate, agg_field, - lst_filters, extra_filters, + col_ref = _try_col_ref(fv_raw) + if col_ref is None: + return None + row_field = col_fields[col_ref] if col_ref < len(col_fields) else "name" + + # Query 1: find link_dt records where target_field op val + try: + matching = frappe.get_all( + link_doctype, + fields=["name"], + filters=[[target_field, comp_op, val]], + limit=0, + ) + matching_names = {str(r["name"]) for r in matching} + except Exception: + return None + + passing_keys = matching_names + failing_keys: set = set() # we only know names of matching; non-matching = absence + # For FRAPPE_GET: absent rows have no value → treat as "no match" (False branch), + # so zero_passes is passed as False below (absent = no match). + return _sql_apply_in_not_in( + doctype, + row_field, + passing_keys, + failing_keys, + False, + want_true, + lst_filters, + plain_extra, + aggregate, + agg_field, + ) + + +def _sql_apply_in_not_in( + doctype, + row_field, + passing_keys, + failing_keys, + zero_passes, + want_true, + lst_filters, + plain_extra, + aggregate, + agg_field, ): - """ - Push formula-column card computation entirely into SQL — zero row fetching. - - Supports (any combination): - =IF(FRAPPE_COUNT("DT", [fk,fv …], A1, …) op N, "t", "f") - =IF(FRAPPE_SUM ("DT","sum_field",[fk,fv …],A1,…) op N, "t","f") - =IF(FRAPPE_AVG (…) op N, "t", "f") - =IF(FRAPPE_GET ("DT", A1, "field") op "val", "t", "f") - =IF(A1 op "val", "t", "f") ← direct column comparison - =IF(A1 op "val", "t", "f") with filter_op "!=" / "contains" / … - - Also handles filter_op other than "=" by inverting the want_true logic. - - Returns the computed value (int/float) or None (caller falls back to row-fetch). - """ - import re - - expr = formula.strip().lstrip("=").strip() - - # ── Require IF(condition, true_val, false_val) ──────────────────────── - parsed = _parse_if_expr(expr) - if not parsed: - return None - cond_str, true_val, false_val = parsed - - # Determine which branch the card filter selects - filter_val_s = str(filter_val).strip() - true_val_s = str(true_val).strip() - false_val_s = str(false_val).strip() - - if filter_op == "=": - if filter_val_s == true_val_s: want_true = True - elif filter_val_s == false_val_s: want_true = False - else: return None - elif filter_op == "!=": - if filter_val_s == true_val_s: want_true = False - elif filter_val_s == false_val_s: want_true = True - else: return None - else: - # contains / numeric ops on formula strings — can't push to SQL - return None - - # Collect plain (non-formula) extra card filters - plain_extra = [ - [ef["col"], ef.get("op", "="), ef.get("val", "")] - for ef in extra_filters - if (ef.get("col") or "").strip() and not ef["col"].startswith("_") - ] - - # ── Dispatch by condition type ──────────────────────────────────────── - return _sql_dispatch_condition( - cond_str, want_true, - doctype, col_fields, aggregate, agg_field, - lst_filters, plain_extra, - ) - - -def _sql_dispatch_condition(cond_str, want_true, doctype, col_fields, - aggregate, agg_field, lst_filters, plain_extra): - """Route a parsed IF-condition to the right SQL handler.""" - import re - - # ── FRAPPE_COUNT / FRAPPE_SUM / FRAPPE_AVG (...) op N ───────────────── - m_agg = re.match( - r'^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG)\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$', - cond_str, re.I | re.S, - ) - if m_agg: - func = m_agg.group(1).upper() - args = _parse_formula_args(m_agg.group(2)) - comp_op = m_agg.group(3).replace("<>", "!=") - try: - threshold = float(m_agg.group(4).strip().strip('"')) - except (ValueError, TypeError): - return None - return _sql_frappe_numeric_func( - func, args, comp_op, threshold, want_true, - doctype, col_fields, aggregate, agg_field, lst_filters, plain_extra, - ) - - # ── FRAPPE_GET("DT", A1, "field") op "val" ──────────────────────────── - m_get = re.match( - r'^FRAPPE_GET\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$', - cond_str, re.I | re.S, - ) - if m_get: - args = _parse_formula_args(m_get.group(1)) - comp_op = m_get.group(2).replace("<>", "!=") - val = _strip_formula_quotes(m_get.group(3).strip()) - return _sql_frappe_get( - args, comp_op, val, want_true, - doctype, col_fields, aggregate, agg_field, lst_filters, plain_extra, - ) - - # ── A1 op "val" — direct column comparison ──────────────────────────── - m_col = re.match(r'^([A-Z]+)1\s*([><=!]+|<>)\s*(.+)$', cond_str, re.I) - if m_col: - idx = _col_letter_to_index(m_col.group(1).upper()) - row_field = col_fields[idx] if idx < len(col_fields) else "name" - comp_op = m_col.group(2).replace("<>", "!=") - val = _strip_formula_quotes(m_col.group(3).strip()) - neg_op = {"=":"!=","!=":"=",">":"<=","<":">=",">=":"<","<=":">"} - op = comp_op if want_true else neg_op.get(comp_op, "!=") - try: - return _count_or_agg( - doctype, - lst_filters + plain_extra + [[row_field, op, val]], - aggregate, agg_field, - ) - except Exception: - return None - - return None - - -def _sql_frappe_numeric_func(func, args, comp_op, threshold, want_true, - doctype, col_fields, aggregate, agg_field, - lst_filters, plain_extra): - """ - Handle IF(FRAPPE_COUNT/SUM/AVG(…) op N, t, f). - Builds a GROUP BY query on the link DocType, resolves passing keys, - then counts/aggregates on the base DocType with IN / NOT IN. - """ - if not args: - return None - - link_doctype = _strip_formula_quotes(args[0]) - if not link_doctype: - return None - - sum_field, arg_start = None, 1 - if func in ("FRAPPE_SUM", "FRAPPE_AVG"): - if len(args) < 2: - return None - sum_field = _strip_formula_quotes(args[1]) - arg_start = 2 - - # Parse fk/fv pairs — exactly one must be a dynamic cell reference (A1) - static_link_filters: list = [] - dynamic_pair = None - - i = arg_start - while i + 1 < len(args): - fk = _strip_formula_quotes(args[i]) - fv_raw = args[i + 1].strip() - - col_ref = _try_col_ref(fv_raw) - if col_ref is not None: - if dynamic_pair is not None: - return None # multiple dynamic cols — can't vectorise - rf = col_fields[col_ref] if col_ref < len(col_fields) else "name" - dynamic_pair = (fk, rf) - else: - # Operator may be embedded in fk: "transaction_date >=" - import re - fk_m = re.match(r'^(.+?)\s+(>=|<=|!=|<>|>|<|=|like)\s*$', fk, re.I) - if fk_m: - static_link_filters.append( - [fk_m.group(1).strip(), fk_m.group(2).strip(), _strip_formula_quotes(fv_raw)] - ) - else: - static_link_filters.append([fk, "=", _strip_formula_quotes(fv_raw)]) - i += 2 - - if dynamic_pair is None: - return None - - link_field, row_field = dynamic_pair - - # ── Query 1: GROUP BY → per-key numeric value ───────────────────────── - try: - if func == "FRAPPE_COUNT": - rows = frappe.db.get_all( - link_doctype, - fields=[link_field, "count(*) as _v"], - filters=static_link_filters, - group_by=link_field, - ) - val_map = {str(r.get(link_field, "")): (r.get("_v") or 0) for r in rows} - - elif func == "FRAPPE_SUM" and sum_field: - rows = frappe.db.get_all( - link_doctype, - fields=[link_field, f"sum({sum_field}) as _v"], - filters=static_link_filters, - group_by=link_field, - ) - val_map = {str(r.get(link_field, "")): float(r.get("_v") or 0) for r in rows} - - elif func == "FRAPPE_AVG" and sum_field: - rows = frappe.db.get_all( - link_doctype, - fields=[link_field, f"sum({sum_field}) as _s", "count(*) as _c"], - filters=static_link_filters, - group_by=link_field, - ) - val_map = { - str(r.get(link_field, "")): float(r.get("_s") or 0) / max(int(r.get("_c") or 1), 1) - for r in rows - } - else: - return None - except Exception: - return None - - # ── Classify keys: passing / failing / absent ───────────────────────── - passing_keys = {k for k, v in val_map.items() if _num_compare(v, comp_op, threshold)} - failing_keys = {k for k in val_map if k not in passing_keys} - zero_passes = _num_compare(0, comp_op, threshold) - - return _sql_apply_in_not_in( - doctype, row_field, passing_keys, failing_keys, zero_passes, - want_true, lst_filters, plain_extra, aggregate, agg_field, - ) - - -def _sql_frappe_get(args, comp_op, val, want_true, - doctype, col_fields, aggregate, agg_field, - lst_filters, plain_extra): - """ - Handle IF(FRAPPE_GET("link_dt", A1, "field") op "val", t, f). - Fetches matching names from link_dt, then IN/NOT IN on base doctype. - """ - if len(args) < 3: - return None - - link_doctype = _strip_formula_quotes(args[0]) - fv_raw = args[1].strip() - target_field = _strip_formula_quotes(args[2]) - - col_ref = _try_col_ref(fv_raw) - if col_ref is None: - return None - row_field = col_fields[col_ref] if col_ref < len(col_fields) else "name" - - # Query 1: find link_dt records where target_field op val - try: - matching = frappe.get_all( - link_doctype, - fields=["name"], - filters=[[target_field, comp_op, val]], - limit=0, - ) - matching_names = {str(r["name"]) for r in matching} - except Exception: - return None - - passing_keys = matching_names - failing_keys: set = set() # we only know names of matching; non-matching = absence - zero_passes = not want_true # rows absent from link_dt → comp_op might not apply - - # For FRAPPE_GET: absent rows have no value → treat as "no match" (False branch) - # zero_passes = False (absent = no match) - return _sql_apply_in_not_in( - doctype, row_field, passing_keys, failing_keys, False, - want_true, lst_filters, plain_extra, aggregate, agg_field, - ) - - -def _sql_apply_in_not_in(doctype, row_field, passing_keys, failing_keys, zero_passes, - want_true, lst_filters, plain_extra, aggregate, agg_field): - """ - Build final base-DocType filters from passing/failing key sets and run aggregate. - - zero_passes = True means rows ABSENT from the link table also satisfy the condition. - want_true = True means we want rows where condition is True. - """ - base = lst_filters + plain_extra - - try: - if want_true: - if zero_passes: - # True = passing_keys ∪ (all rows not in val_map) - # Equivalent: exclude failing_keys - if failing_keys: - return _count_or_agg(doctype, base + [[row_field, "not in", list(failing_keys)]], aggregate, agg_field) - else: - return _count_or_agg(doctype, base, aggregate, agg_field) - else: - # True = only passing_keys - if not passing_keys: - return 0 - return _count_or_agg(doctype, base + [[row_field, "in", list(passing_keys)]], aggregate, agg_field) - else: - if zero_passes: - # False = rows in val_map that don't pass = failing_keys - if not failing_keys: - return 0 - return _count_or_agg(doctype, base + [[row_field, "in", list(failing_keys)]], aggregate, agg_field) - else: - # False = failing_keys ∪ (rows absent from val_map) - # = total − passing_keys - total = frappe.db.count(doctype, filters=base) - if not passing_keys: - return total - passing_count = frappe.db.count(doctype, filters=base + [[row_field, "in", list(passing_keys)]]) - return total - passing_count - except Exception: - return None + """ + Build final base-DocType filters from passing/failing key sets and run aggregate. + + zero_passes = True means rows ABSENT from the link table also satisfy the condition. + want_true = True means we want rows where condition is True. + """ + base = lst_filters + plain_extra + + try: + if want_true: + if zero_passes: + # True = passing_keys union (all rows not in val_map) + # Equivalent: exclude failing_keys + if failing_keys: + return _count_or_agg( + doctype, [*base, [row_field, "not in", list(failing_keys)]], aggregate, agg_field + ) + else: + return _count_or_agg(doctype, base, aggregate, agg_field) + else: + # True = only passing_keys + if not passing_keys: + return 0 + return _count_or_agg( + doctype, [*base, [row_field, "in", list(passing_keys)]], aggregate, agg_field + ) + else: + if zero_passes: + # False = rows in val_map that don't pass = failing_keys + if not failing_keys: + return 0 + return _count_or_agg( + doctype, [*base, [row_field, "in", list(failing_keys)]], aggregate, agg_field + ) + else: + # False = failing_keys union (rows absent from val_map) + # = total - passing_keys + total = frappe.db.count(doctype, filters=base) + if not passing_keys: + return total + passing_count = frappe.db.count( + doctype, filters=[*base, [row_field, "in", list(passing_keys)]] + ) + return total - passing_count + except Exception: + return None def _count_or_agg(doctype, filters, aggregate, agg_field): - """Run COUNT or numeric aggregate on doctype with given filters.""" - if aggregate == "count" or not agg_field: - return frappe.db.count(doctype, filters=filters) - matched = frappe.get_all(doctype, fields=[agg_field], filters=filters, limit=0) - nums = [float(r.get(agg_field) or 0) for r in matched if r.get(agg_field) is not None] - if not nums: - return 0 - if aggregate == "sum": return round(sum(nums), 2) - if aggregate == "avg": return round(sum(nums) / len(nums), 2) - if aggregate == "min": return round(min(nums), 2) - if aggregate == "max": return round(max(nums), 2) - return len(nums) + """Run COUNT or numeric aggregate on doctype with given filters.""" + if aggregate == "count" or not agg_field: + return frappe.db.count(doctype, filters=filters) + matched = frappe.get_all(doctype, fields=[agg_field], filters=filters, limit=0) + nums = [float(r.get(agg_field) or 0) for r in matched if r.get(agg_field) is not None] + if not nums: + return 0 + if aggregate == "sum": + return round(sum(nums), 2) + if aggregate == "avg": + return round(sum(nums) / len(nums), 2) + if aggregate == "min": + return round(min(nums), 2) + if aggregate == "max": + return round(max(nums), 2) + return len(nums) # ── compute_card_aggregate helpers ──────────────────────────────────────────── def _col_letter_to_index(letters: str) -> int: - """'A' → 0, 'B' → 1, 'Z' → 25, 'AA' → 26""" - result = 0 - for c in letters.upper(): - result = result * 26 + (ord(c) - ord("A") + 1) - return result - 1 + """'A' → 0, 'B' → 1, 'Z' → 25, 'AA' → 26""" + result = 0 + for c in letters.upper(): + result = result * 26 + (ord(c) - ord("A") + 1) + return result - 1 def _card_match(cell: str, op: str, val: str) -> bool: - cell, val = cell.strip(), val.strip() - if op == "=": return cell == val - if op == "!=": return cell != val - if op == "contains": return val.lower() in cell.lower() - try: - c, v = float(cell), float(val) - if op == ">": return c > v - if op == "<": return c < v - if op == ">=": return c >= v - if op == "<=": return c <= v - except (ValueError, TypeError): - pass - return False + cell, val = cell.strip(), val.strip() + if op == "=": + return cell == val + if op == "!=": + return cell != val + if op == "contains": + return val.lower() in cell.lower() + try: + c, v = float(cell), float(val) + if op == ">": + return c > v + if op == "<": + return c < v + if op == ">=": + return c >= v + if op == "<=": + return c <= v + except (ValueError, TypeError): + pass + return False def _eval_formula_bulk(formula: str, rows: list, col_fields: list) -> None: - """ - Evaluate a HyperFormula formula template for every row. - Sets row["_val"] in-place. - - Supports nested IFs of arbitrary depth, FRAPPE_FUNC comparisons on - both sides of a condition, and bare FRAPPE_FUNC / column references. - """ - import re - - expr = formula.strip() - if expr.startswith("="): - expr = expr[1:].strip() - - # ── IF(condition, true_val, false_val) — handles nested IFs recursively - parsed_if = _parse_if_expr(expr) - if parsed_if: - cond_str, true_val_str, false_val_str = parsed_if - conds = _eval_condition_bulk(cond_str, rows, col_fields) - true_rows = [row for row, c in zip(rows, conds) if c] - false_rows = [row for row, c in zip(rows, conds) if not c] - if true_rows: - _eval_branch_value(true_val_str, true_rows, col_fields) - if false_rows: - _eval_branch_value(false_val_str, false_rows, col_fields) - return - - # ── Bare FRAPPE_* returning a number ───────────────────────────────── - m = re.match(r'^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$', expr, re.I | re.S) - if m: - func, args_str = m.group(1).upper(), m.group(2) - nums = _frappe_func_bulk(func, _parse_formula_args(args_str), rows, col_fields) - for row, v in zip(rows, nums): - row["_val"] = v - return - - # ── Bare column reference A1 ───────────────────────────────────────── - col_m = re.match(r'^([A-Z]+)1$', expr, re.I) - if col_m: - idx = _col_letter_to_index(col_m.group(1).upper()) - field = col_fields[idx] if idx < len(col_fields) else "name" - for row in rows: - row["_val"] = str(row.get(field, "") or "") - return - - # ── Fallback ───────────────────────────────────────────────────────── - for row in rows: - row["_val"] = "" + """ + Evaluate a HyperFormula formula template for every row. + Sets row["_val"] in-place. + + Supports nested IFs of arbitrary depth, FRAPPE_FUNC comparisons on + both sides of a condition, and bare FRAPPE_FUNC / column references. + """ + import re + + expr = formula.strip() + if expr.startswith("="): + expr = expr[1:].strip() + + # ── IF(condition, true_val, false_val) — handles nested IFs recursively + parsed_if = _parse_if_expr(expr) + if parsed_if: + cond_str, true_val_str, false_val_str = parsed_if + conds = _eval_condition_bulk(cond_str, rows, col_fields) + true_rows = [row for row, c in zip(rows, conds, strict=False) if c] + false_rows = [row for row, c in zip(rows, conds, strict=False) if not c] + if true_rows: + _eval_branch_value(true_val_str, true_rows, col_fields) + if false_rows: + _eval_branch_value(false_val_str, false_rows, col_fields) + return + + # ── Bare FRAPPE_* returning a number ───────────────────────────────── + m = re.match(r"^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$", expr, re.I | re.S) + if m: + func, args_str = m.group(1).upper(), m.group(2) + nums = _frappe_func_bulk(func, _parse_formula_args(args_str), rows, col_fields) + for row, v in zip(rows, nums, strict=False): + row["_val"] = v + return + + # ── Bare column reference A1 ───────────────────────────────────────── + col_m = re.match(r"^([A-Z]+)1$", expr, re.I) + if col_m: + idx = _col_letter_to_index(col_m.group(1).upper()) + field = col_fields[idx] if idx < len(col_fields) else "name" + for row in rows: + row["_val"] = str(row.get(field, "") or "") + return + + # ── Fallback ───────────────────────────────────────────────────────── + for row in rows: + row["_val"] = "" def _eval_branch_value(val_str: str, rows: list, col_fields: list) -> None: - """ - Evaluate one branch of an IF expression (may be a literal, nested IF, - or FRAPPE_FUNC call) and set row["_val"] for each row in the subset. - """ - import re - - val_str = val_str.strip() - - # Nested IF — recurse - if re.match(r'^IF\s*\(', val_str, re.I): - _eval_formula_bulk("=" + val_str, rows, col_fields) - return - - # FRAPPE_FUNC returning a value - fm = re.match(r'^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$', val_str, re.I | re.S) - if fm: - func = fm.group(1).upper() - args = _parse_formula_args(fm.group(2)) - nums = _frappe_func_bulk(func, args, rows, col_fields) - for row, v in zip(rows, nums): - row["_val"] = v - return - - # Literal string / number - literal = _strip_formula_quotes(val_str) - for row in rows: - row["_val"] = literal + """ + Evaluate one branch of an IF expression (may be a literal, nested IF, + or FRAPPE_FUNC call) and set row["_val"] for each row in the subset. + """ + import re + + val_str = val_str.strip() + + # Nested IF — recurse + if re.match(r"^IF\s*\(", val_str, re.I): + _eval_formula_bulk("=" + val_str, rows, col_fields) + return + + # FRAPPE_FUNC returning a value + fm = re.match(r"^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$", val_str, re.I | re.S) + if fm: + func = fm.group(1).upper() + args = _parse_formula_args(fm.group(2)) + nums = _frappe_func_bulk(func, args, rows, col_fields) + for row, v in zip(rows, nums, strict=False): + row["_val"] = v + return + + # Literal string / number + literal = _strip_formula_quotes(val_str) + for row in rows: + row["_val"] = literal def _parse_if_expr(expr: str): - """ - Parse ``IF(cond, true, false)`` → (cond_str, true_val, false_val) or None. - """ - import re - if not re.match(r'^IF\s*\(', expr, re.I): - return None - # Strip leading "IF(" - inner = re.sub(r'^IF\s*\(\s*', "", expr, flags=re.I) - # Remove trailing ")" - if inner.endswith(")"): - inner = inner[:-1] - parts = _parse_formula_args(inner) - if len(parts) < 3: - return None - return ( - parts[0].strip(), - _strip_formula_quotes(parts[1].strip()), - _strip_formula_quotes(parts[2].strip()), - ) + """ + Parse ``IF(cond, true, false)`` → (cond_str, true_val, false_val) or None. + """ + import re + + if not re.match(r"^IF\s*\(", expr, re.I): + return None + # Strip leading "IF(" + inner = re.sub(r"^IF\s*\(\s*", "", expr, flags=re.I) + # Remove trailing ")" + if inner.endswith(")"): + inner = inner[:-1] + parts = _parse_formula_args(inner) + if len(parts) < 3: + return None + return ( + parts[0].strip(), + _strip_formula_quotes(parts[1].strip()), + _strip_formula_quotes(parts[2].strip()), + ) def _eval_condition_bulk(cond_str: str, rows: list, col_fields: list) -> list: - """Evaluate a condition string for all rows, returning list[bool].""" - import re - - # FRAPPE_FUNC(...) op threshold - fm = re.match( - r'^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$', - cond_str, re.I | re.S, - ) - if fm: - func = fm.group(1).upper() - args = _parse_formula_args(fm.group(2)) - op = fm.group(3).replace("<>", "!=") - rhs = fm.group(4).strip() - lhs_nums = _frappe_func_bulk(func, args, rows, col_fields) - - # RHS may be another FRAPPE_FUNC (e.g. FRAPPE_SUM(...) >= FRAPPE_SUM(...)) - rhs_fm = re.match( - r'^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$', - rhs, re.I | re.S, - ) - if rhs_fm: - rhs_func = rhs_fm.group(1).upper() - rhs_args = _parse_formula_args(rhs_fm.group(2)) - rhs_nums = _frappe_func_bulk(rhs_func, rhs_args, rows, col_fields) - return [_num_compare(l, op, r) for l, r in zip(lhs_nums, rhs_nums)] - - try: - threshold = float(rhs.strip('"')) - except (ValueError, TypeError): - threshold = 0.0 - return [_num_compare(v, op, threshold) for v in lhs_nums] - - # Column reference op literal (A1 = "value") - cm = re.match(r'^([A-Z]+)1\s*([><=!]+|<>)\s*(.+)$', cond_str, re.I) - if cm: - idx = _col_letter_to_index(cm.group(1).upper()) - field = col_fields[idx] if idx < len(col_fields) else "name" - op = cm.group(2).replace("<>", "!=") - val = _strip_formula_quotes(cm.group(3).strip()) - return [_card_match(str(r.get(field, "") or ""), op, val) for r in rows] - - return [True] * len(rows) + """Evaluate a condition string for all rows, returning list[bool].""" + import re + + # FRAPPE_FUNC(...) op threshold + fm = re.match( + r"^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)\s*([><=!]+|<>)\s*(.+)$", + cond_str, + re.I | re.S, + ) + if fm: + func = fm.group(1).upper() + args = _parse_formula_args(fm.group(2)) + op = fm.group(3).replace("<>", "!=") + rhs = fm.group(4).strip() + lhs_nums = _frappe_func_bulk(func, args, rows, col_fields) + + # RHS may be another FRAPPE_FUNC (e.g. FRAPPE_SUM(...) >= FRAPPE_SUM(...)) + rhs_fm = re.match( + r"^(FRAPPE_COUNT|FRAPPE_SUM|FRAPPE_AVG|FRAPPE_GET)\s*\((.+)\)$", + rhs, + re.I | re.S, + ) + if rhs_fm: + rhs_func = rhs_fm.group(1).upper() + rhs_args = _parse_formula_args(rhs_fm.group(2)) + rhs_nums = _frappe_func_bulk(rhs_func, rhs_args, rows, col_fields) + return [_num_compare(l, op, r) for l, r in zip(lhs_nums, rhs_nums, strict=False)] + + try: + threshold = float(rhs.strip('"')) + except (ValueError, TypeError): + threshold = 0.0 + return [_num_compare(v, op, threshold) for v in lhs_nums] + + # Column reference op literal (A1 = "value") + cm = re.match(r"^([A-Z]+)1\s*([><=!]+|<>)\s*(.+)$", cond_str, re.I) + if cm: + idx = _col_letter_to_index(cm.group(1).upper()) + field = col_fields[idx] if idx < len(col_fields) else "name" + op = cm.group(2).replace("<>", "!=") + val = _strip_formula_quotes(cm.group(3).strip()) + return [_card_match(str(r.get(field, "") or ""), op, val) for r in rows] + + return [True] * len(rows) def _frappe_func_bulk(func: str, args: list, rows: list, col_fields: list) -> list: - """ - Evaluate FRAPPE_COUNT / FRAPPE_SUM / FRAPPE_AVG for all rows at once - using a single GROUP BY query when possible. - """ - import re - - if not args: - return [0] * len(rows) - - link_doctype = _strip_formula_quotes(args[0]) - if not link_doctype: - return [0] * len(rows) - - # FRAPPE_SUM / FRAPPE_AVG: 2nd arg is the sum field - sum_field = None - arg_start = 1 - if func in ("FRAPPE_SUM", "FRAPPE_AVG"): - if len(args) < 2: - return [0] * len(rows) - sum_field = _strip_formula_quotes(args[1]) - arg_start = 2 - - # Parse fk/fv pairs — fv may be a column ref (A1) or a string literal - static_filters: list = [] - dynamic_pairs: list = [] # (link_field, row_fieldname) - - i = arg_start - while i + 1 < len(args): - fk = _strip_formula_quotes(args[i]) - fv_raw = args[i + 1].strip() - - # Handle operator-embedded fieldnames: "transaction_date >=" → field="transaction_date", op=">=" - fk_op = "=" - fk_m = re.match(r'^(.+?)\s+(>=|<=|!=|<>|>|<|=|like)\s*$', fk, re.I) - if fk_m: - fk, fk_op = fk_m.group(1).strip(), fk_m.group(2).strip() - - col_ref = _try_col_ref(fv_raw) - if col_ref is not None: - rf = col_fields[col_ref] if col_ref < len(col_fields) else "name" - dynamic_pairs.append((fk, rf)) - else: - static_filters.append([fk, fk_op, _strip_formula_quotes(fv_raw)]) - i += 2 - - # All-static: same value for every row - if not dynamic_pairs: - try: - cnt = frappe.db.count(link_doctype, filters=static_filters) or 0 - except Exception: - cnt = 0 - return [cnt] * len(rows) - - # Single dynamic field: one GROUP BY query covers all rows - if len(dynamic_pairs) == 1: - dyn_fk, dyn_rf = dynamic_pairs[0] - row_vals = [r.get(dyn_rf) for r in rows] - unique = list({v for v in row_vals if v is not None}) - if not unique: - return [0] * len(rows) - - bulk_filters = static_filters + [[dyn_fk, "in", unique]] - val_map: dict = {} - try: - if func == "FRAPPE_COUNT": - agg_rows = frappe.db.get_all( - link_doctype, - fields=[dyn_fk, "count(*) as cnt"], - filters=bulk_filters, - group_by=dyn_fk, - ) - val_map = {str(r.get(dyn_fk, "")): (r.get("cnt") or 0) for r in agg_rows} - elif func == "FRAPPE_SUM" and sum_field: - agg_rows = frappe.db.get_all( - link_doctype, - fields=[dyn_fk, f"sum({sum_field}) as s"], - filters=bulk_filters, - group_by=dyn_fk, - ) - val_map = {str(r.get(dyn_fk, "")): float(r.get("s") or 0) for r in agg_rows} - elif func == "FRAPPE_AVG" and sum_field: - agg_rows = frappe.db.get_all( - link_doctype, - fields=[dyn_fk, f"sum({sum_field}) as s", "count(*) as cnt"], - filters=bulk_filters, - group_by=dyn_fk, - ) - val_map = { - str(r.get(dyn_fk, "")): ( - float(r.get("s") or 0) / max(int(r.get("cnt") or 1), 1) - ) - for r in agg_rows - } - except Exception: - pass - - return [val_map.get(str(rv), 0) for rv in row_vals] - - # Multiple dynamic fields: row-by-row (rare) - results = [] - for row in rows: - dyn_f = [[fk, "=", row.get(rf)] for fk, rf in dynamic_pairs] - all_f = static_filters + dyn_f - try: - if func == "FRAPPE_COUNT": - v: float = float(frappe.db.count(link_doctype, filters=all_f) or 0) - elif func in ("FRAPPE_SUM", "FRAPPE_AVG") and sum_field: - r_rows = frappe.db.get_all(link_doctype, fields=[sum_field], filters=all_f) - nums = [float(x[sum_field] or 0) for x in r_rows if x.get(sum_field) is not None] - v = sum(nums) if func == "FRAPPE_SUM" else (sum(nums) / len(nums) if nums else 0.0) - else: - v = 0.0 - except Exception: - v = 0.0 - results.append(v) - return results + """ + Evaluate FRAPPE_COUNT / FRAPPE_SUM / FRAPPE_AVG for all rows at once + using a single GROUP BY query when possible. + """ + import re + + if not args: + return [0] * len(rows) + + link_doctype = _strip_formula_quotes(args[0]) + if not link_doctype: + return [0] * len(rows) + + # FRAPPE_SUM / FRAPPE_AVG: 2nd arg is the sum field + sum_field = None + arg_start = 1 + if func in ("FRAPPE_SUM", "FRAPPE_AVG"): + if len(args) < 2: + return [0] * len(rows) + sum_field = _strip_formula_quotes(args[1]) + arg_start = 2 + + # Parse fk/fv pairs — fv may be a column ref (A1) or a string literal + static_filters: list = [] + dynamic_pairs: list = [] # (link_field, row_fieldname) + + i = arg_start + while i + 1 < len(args): + fk = _strip_formula_quotes(args[i]) + fv_raw = args[i + 1].strip() + + # Handle operator-embedded fieldnames: "transaction_date >=" → field="transaction_date", op=">=" + fk_op = "=" + fk_m = re.match(r"^(.+?)\s+(>=|<=|!=|<>|>|<|=|like)\s*$", fk, re.I) + if fk_m: + fk, fk_op = fk_m.group(1).strip(), fk_m.group(2).strip() + + col_ref = _try_col_ref(fv_raw) + if col_ref is not None: + rf = col_fields[col_ref] if col_ref < len(col_fields) else "name" + dynamic_pairs.append((fk, rf)) + else: + static_filters.append([fk, fk_op, _strip_formula_quotes(fv_raw)]) + i += 2 + + # All-static: same value for every row + if not dynamic_pairs: + try: + cnt = frappe.db.count(link_doctype, filters=static_filters) or 0 + except Exception: + cnt = 0 + return [cnt] * len(rows) + + # Single dynamic field: one GROUP BY query covers all rows + if len(dynamic_pairs) == 1: + dyn_fk, dyn_rf = dynamic_pairs[0] + row_vals = [r.get(dyn_rf) for r in rows] + unique = list({v for v in row_vals if v is not None}) + if not unique: + return [0] * len(rows) + + bulk_filters = [*static_filters, [dyn_fk, "in", unique]] + val_map: dict = {} + try: + if func == "FRAPPE_COUNT": + agg_rows = frappe.db.get_all( + link_doctype, + fields=[dyn_fk, "count(*) as cnt"], + filters=bulk_filters, + group_by=dyn_fk, + ) + val_map = {str(r.get(dyn_fk, "")): (r.get("cnt") or 0) for r in agg_rows} + elif func == "FRAPPE_SUM" and sum_field: + agg_rows = frappe.db.get_all( + link_doctype, + fields=[dyn_fk, f"sum({sum_field}) as s"], + filters=bulk_filters, + group_by=dyn_fk, + ) + val_map = {str(r.get(dyn_fk, "")): float(r.get("s") or 0) for r in agg_rows} + elif func == "FRAPPE_AVG" and sum_field: + agg_rows = frappe.db.get_all( + link_doctype, + fields=[dyn_fk, f"sum({sum_field}) as s", "count(*) as cnt"], + filters=bulk_filters, + group_by=dyn_fk, + ) + val_map = { + str(r.get(dyn_fk, "")): (float(r.get("s") or 0) / max(int(r.get("cnt") or 1), 1)) + for r in agg_rows + } + except Exception: + pass + + return [val_map.get(str(rv), 0) for rv in row_vals] + + # Multiple dynamic fields: row-by-row (rare) + results = [] + for row in rows: + dyn_f = [[fk, "=", row.get(rf)] for fk, rf in dynamic_pairs] + all_f = static_filters + dyn_f + try: + if func == "FRAPPE_COUNT": + v: float = float(frappe.db.count(link_doctype, filters=all_f) or 0) + elif func in ("FRAPPE_SUM", "FRAPPE_AVG") and sum_field: + r_rows = frappe.db.get_all(link_doctype, fields=[sum_field], filters=all_f) + nums = [float(x[sum_field] or 0) for x in r_rows if x.get(sum_field) is not None] + v = sum(nums) if func == "FRAPPE_SUM" else (sum(nums) / len(nums) if nums else 0.0) + else: + v = 0.0 + except Exception: + v = 0.0 + results.append(v) + return results def _parse_formula_args(args_str: str) -> list: - """Split formula args on commas, respecting quoted strings and nested parens.""" - parts, current, depth, in_q = [], [], 0, False - for ch in args_str: - if ch == '"' and depth == 0: - in_q = not in_q - current.append(ch) - elif not in_q: - if ch == "(": - depth += 1; current.append(ch) - elif ch == ")": - depth -= 1; current.append(ch) - elif ch == "," and depth == 0: - parts.append("".join(current)) - current = [] - else: - current.append(ch) - else: - current.append(ch) - if current: - parts.append("".join(current)) - return parts + """Split formula args on commas, respecting quoted strings and nested parens.""" + parts, current, depth, in_q = [], [], 0, False + for ch in args_str: + if ch == '"' and depth == 0: + in_q = not in_q + current.append(ch) + elif not in_q: + if ch == "(": + depth += 1 + current.append(ch) + elif ch == ")": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + parts.append("".join(current)) + current = [] + else: + current.append(ch) + else: + current.append(ch) + if current: + parts.append("".join(current)) + return parts def _strip_formula_quotes(s: str) -> str: - s = s.strip() - if len(s) >= 2 and s[0] == '"' and s[-1] == '"': - return s[1:-1] - return s + s = s.strip() + if len(s) >= 2 and s[0] == '"' and s[-1] == '"': + return s[1:-1] + return s def _try_col_ref(s: str): - """Return column index if s matches A1, B1, AA1 etc., else None.""" - import re - m = re.match(r'^([A-Z]+)1$', s.strip(), re.I) - return _col_letter_to_index(m.group(1).upper()) if m else None + """Return column index if s matches A1, B1, AA1 etc., else None.""" + import re + + m = re.match(r"^([A-Z]+)1$", s.strip(), re.I) + return _col_letter_to_index(m.group(1).upper()) if m else None def _num_compare(val, op: str, threshold: float) -> bool: - try: - v = float(val) - if op == ">": return v > threshold - if op == "<": return v < threshold - if op == ">=": return v >= threshold - if op == "<=": return v <= threshold - if op == "=": return v == threshold - if op == "!=": return v != threshold - except (TypeError, ValueError): - pass - return False + try: + v = float(val) + if op == ">": + return v > threshold + if op == "<": + return v < threshold + if op == ">=": + return v >= threshold + if op == "<=": + return v <= threshold + if op == "=": + return v == threshold + if op == "!=": + return v != threshold + except (TypeError, ValueError): + pass + return False @frappe.whitelist() def gl_balance( - account: str, - company: str, - from_date: str | None = None, - to_date: str | None = None, - cost_center: str | None = None, - finance_book: str | None = None, + account: str, + company: str, + from_date: str | None = None, + to_date: str | None = None, + cost_center: str | None = None, + finance_book: str | None = None, ) -> dict: - """ - Return the net GL balance (debit − credit) for an account/company/period. - - Used by the ``GL_BALANCE(account, company, …)`` HyperFormula function. - Returns ``{"value": 0, "note": "…"}`` if ERPNext is not installed so the - formula degrades gracefully on vanilla Frappe setups. - - Args: - account: GL account name (e.g. "Cash - ACME"). - company: Company name. - from_date: Start of period (inclusive). None = no lower bound. - to_date: End of period (inclusive). None = no upper bound. - cost_center: Optional cost-centre filter. - finance_book: Optional finance-book filter (also matches IS NULL rows). - - Returns: - {"value": } - """ - if not frappe.db.table_exists("tabGL Entry"): - return {"value": 0, "note": "GL Entry not available — ERPNext not installed"} - - frappe.has_permission("GL Entry", "read", throw=True) - - conditions = [ - "account = %(account)s", - "company = %(company)s", - "is_cancelled = 0", - ] - params: dict = {"account": account, "company": company} - - if from_date: - conditions.append("posting_date >= %(from_date)s") - params["from_date"] = from_date - if to_date: - conditions.append("posting_date <= %(to_date)s") - params["to_date"] = to_date - if cost_center: - conditions.append("cost_center = %(cost_center)s") - params["cost_center"] = cost_center - if finance_book: - conditions.append("(finance_book = %(finance_book)s OR finance_book IS NULL)") - params["finance_book"] = finance_book - - where = " AND ".join(conditions) - result = frappe.db.sql( - f"SELECT COALESCE(SUM(debit), 0) - COALESCE(SUM(credit), 0) AS balance " - f"FROM `tabGL Entry` WHERE {where}", - params, - as_dict=True, - ) - - return {"value": float(result[0].get("balance") or 0)} + """ + Return the net GL balance (debit - credit) for an account/company/period. + + Used by the ``GL_BALANCE(account, company, …)`` HyperFormula function. + Returns ``{"value": 0, "note": "…"}`` if ERPNext is not installed so the + formula degrades gracefully on vanilla Frappe setups. + + Args: + account: GL account name (e.g. "Cash - ACME"). + company: Company name. + from_date: Start of period (inclusive). None = no lower bound. + to_date: End of period (inclusive). None = no upper bound. + cost_center: Optional cost-centre filter. + finance_book: Optional finance-book filter (also matches IS NULL rows). + + Returns: + {"value": } + """ + if not frappe.db.table_exists("tabGL Entry"): + return {"value": 0, "note": "GL Entry not available — ERPNext not installed"} + + frappe.has_permission("GL Entry", "read", throw=True) + + conditions = [ + "account = %(account)s", + "company = %(company)s", + "is_cancelled = 0", + ] + params: dict = {"account": account, "company": company} + + if from_date: + conditions.append("posting_date >= %(from_date)s") + params["from_date"] = from_date + if to_date: + conditions.append("posting_date <= %(to_date)s") + params["to_date"] = to_date + if cost_center: + conditions.append("cost_center = %(cost_center)s") + params["cost_center"] = cost_center + if finance_book: + conditions.append("(finance_book = %(finance_book)s OR finance_book IS NULL)") + params["finance_book"] = finance_book + + where = " AND ".join(conditions) + result = frappe.db.sql( + f"SELECT COALESCE(SUM(debit), 0) - COALESCE(SUM(credit), 0) AS balance " + f"FROM `tabGL Entry` WHERE {where}", + params, + as_dict=True, + ) + + return {"value": float(result[0].get("balance") or 0)} @frappe.whitelist() def stock_qty( - item_code: str, - warehouse: str, - as_of_date: str | None = None, + item_code: str, + warehouse: str, + as_of_date: str | None = None, ) -> dict: - """ - Return current or point-in-time stock quantity for an item/warehouse. - - Used by ``STOCK_QTY(item_code, warehouse [, as_of_date])``. - When *as_of_date* is omitted the Bin table is used (fast current balance). - When a date is supplied the Stock Ledger Entry table is queried to return - the running total up to that date. - - Returns: - {"value": } - """ - if not frappe.db.table_exists("tabBin"): - return {"value": 0, "note": "Bin doctype not available — ERPNext not installed"} - - frappe.has_permission("Bin", "read", throw=True) - - if not as_of_date: - qty = frappe.db.get_value( - "Bin", - {"item_code": item_code, "warehouse": warehouse}, - "actual_qty", - ) - return {"value": float(qty or 0)} - - # Point-in-time balance via Stock Ledger Entry - result = frappe.db.sql( - """ + """ + Return current or point-in-time stock quantity for an item/warehouse. + + Used by ``STOCK_QTY(item_code, warehouse [, as_of_date])``. + When *as_of_date* is omitted the Bin table is used (fast current balance). + When a date is supplied the Stock Ledger Entry table is queried to return + the running total up to that date. + + Returns: + {"value": } + """ + if not frappe.db.table_exists("tabBin"): + return {"value": 0, "note": "Bin doctype not available — ERPNext not installed"} + + frappe.has_permission("Bin", "read", throw=True) + + if not as_of_date: + qty = frappe.db.get_value( + "Bin", + {"item_code": item_code, "warehouse": warehouse}, + "actual_qty", + ) + return {"value": float(qty or 0)} + + # Point-in-time balance via Stock Ledger Entry + result = frappe.db.sql( + """ SELECT COALESCE(SUM(actual_qty), 0) AS qty FROM `tabStock Ledger Entry` WHERE item_code = %(item)s @@ -2331,78 +2586,78 @@ def stock_qty( AND posting_date <= %(date)s AND docstatus = 1 """, - {"item": item_code, "wh": warehouse, "date": as_of_date}, - as_dict=True, - ) - return {"value": float(result[0].get("qty") or 0)} + {"item": item_code, "wh": warehouse, "date": as_of_date}, + as_dict=True, + ) + return {"value": float(result[0].get("qty") or 0)} @frappe.whitelist() def item_price( - item_code: str, - price_list: str, - qty: float | None = None, - customer: str | None = None, - uom: str | None = None, + item_code: str, + price_list: str, + qty: float | None = None, + customer: str | None = None, + uom: str | None = None, ) -> dict: - """ - Return the selling price for an item from an Item Price record. - - Used by ``ITEM_PRICE(item_code, price_list [, qty, customer, uom])``. - Picks the most recently valid price (valid_from ≤ today ≤ valid_upto). - Falls back to the most recently modified price if no date-valid record - exists. - - Args: - item_code: Item code. - price_list: Price list name (e.g. "Standard Selling"). - qty: Quantity (reserved for future tiered pricing — unused now). - customer: Customer (reserved for future customer-group pricing). - uom: Unit of measure filter. - - Returns: - {"value": } - """ - if not frappe.db.table_exists("tabItem Price"): - return {"value": 0, "note": "Item Price not available — ERPNext not installed"} - - frappe.has_permission("Item Price", "read", throw=True) - - filters: dict = { - "item_code": item_code, - "price_list": price_list, - "selling": 1, - } - if uom: - filters["uom"] = uom - - prices = frappe.get_list( - "Item Price", - filters=filters, - fields=["price_list_rate", "valid_from", "valid_upto"], - order_by="valid_from desc", - ) - - if not prices: - return {"value": 0} - - today = frappe.utils.today() - - for p in prices: - valid_from = p.get("valid_from") - valid_upto = p.get("valid_upto") - - # Skip if period hasn't started yet - if valid_from and frappe.utils.getdate(valid_from) > frappe.utils.getdate(today): - continue - # Skip if period has already ended - if valid_upto and frappe.utils.getdate(valid_upto) < frappe.utils.getdate(today): - continue - - return {"value": float(p.get("price_list_rate") or 0)} - - # Fallback: first record (most recent valid_from regardless of date range) - return {"value": float(prices[0].get("price_list_rate") or 0)} + """ + Return the selling price for an item from an Item Price record. + + Used by ``ITEM_PRICE(item_code, price_list [, qty, customer, uom])``. + Picks the most recently valid price (valid_from ≤ today ≤ valid_upto). + Falls back to the most recently modified price if no date-valid record + exists. + + Args: + item_code: Item code. + price_list: Price list name (e.g. "Standard Selling"). + qty: Quantity (reserved for future tiered pricing — unused now). + customer: Customer (reserved for future customer-group pricing). + uom: Unit of measure filter. + + Returns: + {"value": } + """ + if not frappe.db.table_exists("tabItem Price"): + return {"value": 0, "note": "Item Price not available — ERPNext not installed"} + + frappe.has_permission("Item Price", "read", throw=True) + + filters: dict = { + "item_code": item_code, + "price_list": price_list, + "selling": 1, + } + if uom: + filters["uom"] = uom + + prices = frappe.get_list( + "Item Price", + filters=filters, + fields=["price_list_rate", "valid_from", "valid_upto"], + order_by="valid_from desc", + ) + + if not prices: + return {"value": 0} + + today = frappe.utils.today() + + for p in prices: + valid_from = p.get("valid_from") + valid_upto = p.get("valid_upto") + + # Skip if period hasn't started yet + if valid_from and frappe.utils.getdate(valid_from) > frappe.utils.getdate(today): + continue + # Skip if period has already ended + if valid_upto and frappe.utils.getdate(valid_upto) < frappe.utils.getdate(today): + continue + + return {"value": float(p.get("price_list_rate") or 0)} + + # Fallback: first record (most recent valid_from regardless of date range) + return {"value": float(prices[0].get("price_list_rate") or 0)} # ── Read ────────────────────────────────────────────────────────────────────── @@ -2491,18 +2746,18 @@ def save_workbook( doc = frappe.new_doc("Excel Workbook") doc.doctype_name = doctype_name - doc.title = title - doc.is_public = frappe.utils.cint(is_public) - doc.columns_config = columns_config + doc.title = title + doc.is_public = frappe.utils.cint(is_public) + doc.columns_config = columns_config doc.formula_columns = formula_columns - doc.filters = filters - doc.sort_by = sort_by or "{}" - doc.join_config = join_config or "{}" - doc.sheets = sheets or "[]" - doc.chart_overlays = chart_overlays or "[]" - doc.format_store = format_store or "{}" - doc.cond_fmt_rules = cond_fmt_rules or "[]" - doc.view_state = view_state or "{}" + doc.filters = filters + doc.sort_by = sort_by or "{}" + doc.join_config = join_config or "{}" + doc.sheets = sheets or "[]" + doc.chart_overlays = chart_overlays or "[]" + doc.format_store = format_store or "{}" + doc.cond_fmt_rules = cond_fmt_rules or "[]" + doc.view_state = view_state or "{}" doc.save(ignore_permissions=False) @@ -2534,6 +2789,7 @@ def delete_workbook(name: str) -> dict: # ── Shared helpers ──────────────────────────────────────────────────────────── + def _sample_and_profile(doctype: str, field: str, limit: int = 100) -> dict: """ Sample `limit` values from doctype.field and detect their structural pattern. @@ -2564,15 +2820,15 @@ def _sample_and_profile(doctype: str, field: str, limit: int = 100) -> dict: def _ratio(fn): return sum(1 for v in s if fn(v)) / n - series_re = re.compile(r'^([A-Z][A-Z0-9-]*)-(?:\d{4}-)?[0-9]+$') + series_re = re.compile(r"^([A-Z][A-Z0-9-]*)-(?:\d{4}-)?[0-9]+$") - if _ratio(lambda v: bool(re.match(r'^[^@\s]+@[^@\s]+\.[^@\s]+$', v))) >= 0.8: + if _ratio(lambda v: bool(re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", v))) >= 0.8: return {"values": set(raw), "raw_list": raw, "pattern": "email", "prefix_set": set()} - if _ratio(lambda v: bool(re.match(r'^\d{4}-\d{2}-\d{2}', v))) >= 0.8: + if _ratio(lambda v: bool(re.match(r"^\d{4}-\d{2}-\d{2}", v))) >= 0.8: return {"values": set(raw), "raw_list": raw, "pattern": "date", "prefix_set": set()} - if _ratio(lambda v: bool(re.match(r'^-?\d+\.?\d*$', v))) >= 0.8: + if _ratio(lambda v: bool(re.match(r"^-?\d+\.?\d*$", v))) >= 0.8: return {"values": set(raw), "raw_list": raw, "pattern": "numeric", "prefix_set": set()} series_matches = [series_re.match(v) for v in s] @@ -2597,40 +2853,54 @@ def _l1_pattern_score(src_prof: dict, tgt_prof: dict) -> float: return 0.95 return 0.80 - if sp == "hash" and tp == "hash": return 0.70 - if sp == "email" and tp == "email": return 0.60 - if sp == tp: return 0.40 # same generic type - return 0.10 # different + if sp == "hash" and tp == "hash": + return 0.70 + if sp == "email" and tp == "email": + return 0.60 + if sp == tp: + return 0.40 # same generic type + return 0.10 # different # Hard incompatible pattern pairs — type-gate (Layer 2) _INCOMPATIBLE_PAIRS = { - ("date", "numeric"), ("numeric", "date"), - ("date", "email"), ("email", "date"), - ("numeric", "email"), ("email", "numeric"), - ("numeric", "naming_series"), ("naming_series", "numeric"), - ("numeric", "hash"), ("hash", "numeric"), + ("date", "numeric"), + ("numeric", "date"), + ("date", "email"), + ("email", "date"), + ("numeric", "email"), + ("email", "numeric"), + ("numeric", "naming_series"), + ("naming_series", "numeric"), + ("numeric", "hash"), + ("hash", "numeric"), } def _grade(composite: float, method: str) -> str: - if method == "meta": return "S" - if composite >= 0.80: return "A" - if composite >= 0.60: return "B" - if composite >= 0.40: return "C" - if composite >= 0.25: return "D" + if method == "meta": + return "S" + if composite >= 0.80: + return "A" + if composite >= 0.60: + return "B" + if composite >= 0.40: + return "C" + if composite >= 0.25: + return "D" return "F" def _cardinality_and_coverage(src_list: list, tgt_list: list): """Returns (cardinality_str, coverage_float).""" from collections import Counter + src_set = set(src_list) tgt_set = set(tgt_list) coverage = round(len(src_set & tgt_set) / max(len(src_set), 1), 2) - tgt_cnt = Counter(tgt_list) - matched = [v for v in src_list if v in tgt_set] - max_m = max((tgt_cnt.get(v, 0) for v in matched), default=0) + tgt_cnt = Counter(tgt_list) + matched = [v for v in src_list if v in tgt_set] + max_m = max((tgt_cnt.get(v, 0) for v in matched), default=0) return ("1:1" if max_m <= 1 else "1:N"), coverage @@ -2815,11 +3085,13 @@ def _get_dynamic_link_edges() -> list: try: target_meta = frappe.get_meta(target_dt) if not target_meta.issingle and not target_meta.istable: - targets_data.append({ - "target_doctype": target_dt, - "count": None, # No count for Select (all are possible) - "method": "select" - }) + targets_data.append( + { + "target_doctype": target_dt, + "count": None, # No count for Select (all are possible) + "method": "select", + } + ) except Exception: continue # Meta fetch failed - skip @@ -2831,11 +3103,7 @@ def _get_dynamic_link_edges() -> list: WHERE `{ref_field}` IS NOT NULL AND `{ref_field}` != '' GROUP BY `{ref_field}` LIMIT {limit} - """.format( - doctype=df["doctype"], - ref_field=df["ref_field"], - limit=_DYNAMIC_LINK_SAMPLE_LIMIT - ) + """.format(doctype=df["doctype"], ref_field=df["ref_field"], limit=_DYNAMIC_LINK_SAMPLE_LIMIT) sampled_targets = frappe.db.sql(sample_sql, as_dict=True) @@ -2849,28 +3117,32 @@ def _get_dynamic_link_edges() -> list: try: target_meta = frappe.get_meta(row["target_doctype"]) if not target_meta.issingle and not target_meta.istable: - targets_data.append({ - "target_doctype": row["target_doctype"], - "count": row["count"], - "method": "data_sample" - }) + targets_data.append( + { + "target_doctype": row["target_doctype"], + "count": row["count"], + "method": "data_sample", + } + ) except Exception: continue # Create edges for discovered targets for target_info in targets_data: - edges.append({ - "doctype": df["doctype"], - "fieldname": df["fieldname"], - "label": df["label"], - "target": target_info["target_doctype"], - "is_child_src": df["is_child_src"], - "is_dynamic": True, - "ref_field": df["ref_field"], - "ref_field_type": ref_field_obj.fieldtype, - "detection_method": target_info["method"], - "sample_count": target_info.get("count") - }) + edges.append( + { + "doctype": df["doctype"], + "fieldname": df["fieldname"], + "label": df["label"], + "target": target_info["target_doctype"], + "is_child_src": df["is_child_src"], + "is_dynamic": True, + "ref_field": df["ref_field"], + "ref_field_type": ref_field_obj.fieldtype, + "detection_method": target_info["method"], + "sample_count": target_info.get("count"), + } + ) except Exception: # If table doesn't exist or meta fetch fails, skip this Dynamic Link @@ -2882,9 +3154,11 @@ def _get_dynamic_link_edges() -> list: # ── Main validation endpoint ────────────────────────────────────────────────── + def _csv_to_headers_rows(text: str): """Parse CSV text → {"headers": [...], "rows": [[...]]}.""" - import csv, io + import csv + import io reader = csv.reader(io.StringIO(text)) all_rows = list(reader) @@ -2893,10 +3167,11 @@ def _csv_to_headers_rows(text: str): return {"headers": all_rows[0], "rows": all_rows[1:]} - def _validate_external_url(url: str) -> None: """Block SSRF: reject requests to RFC-1918 / loopback / link-local addresses.""" - import ipaddress, socket, re + import ipaddress + import re + import socket from urllib.parse import urlparse parsed = urlparse(url) @@ -2933,7 +3208,9 @@ def fetch_google_sheet(url: str, tab_name: str = ""): Uses the CSV export endpoint — no API key needed for sheets shared as "Anyone with link can view". """ - import re, requests + import re + + import requests match = re.search(r"/spreadsheets/d/([a-zA-Z0-9_-]+)", url) if not match: @@ -2943,6 +3220,7 @@ def fetch_google_sheet(url: str, tab_name: str = ""): export_url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv" if tab_name: from urllib.parse import quote + export_url += f"&sheet={quote(tab_name)}" try: @@ -2980,7 +3258,9 @@ def fetch_web_api(url: str, method: str = "GET", headers: str = "", json_path: s headers: JSON-encoded dict of request headers. json_path: Dot-notation path into the JSON response (e.g. "data.items"). """ - import json, requests + import json + + import requests _validate_external_url(url) hdrs: dict = {} @@ -3035,7 +3315,8 @@ def extract_pdf_tables(pdf_b64: str): Returns {"tables": [{"headers": [...], "rows": [[...], ...]}, ...]}. """ - import base64, io + import base64 + import io try: import pdfplumber @@ -3047,7 +3328,7 @@ def extract_pdf_tables(pdf_b64: str): tables = [] with pdfplumber.open(io.BytesIO(pdf_bytes)) as pdf: for page in pdf.pages: - for tbl in (page.extract_tables() or []): + for tbl in page.extract_tables() or []: if not tbl: continue headers = [str(c or f"Col{i + 1}") for i, c in enumerate(tbl[0])] @@ -3059,13 +3340,11 @@ def extract_pdf_tables(pdf_b64: str): return {"error": str(e)} - - # ── Field-type compatibility helpers ───────────────────────────────────────── _NUMERIC_FT: frozenset = frozenset({"Currency", "Float", "Int", "Percent"}) -_TEXT_FT: frozenset = frozenset({"Data", "Link", "Select", "Small Text", "Text", "Long Text"}) -_DATE_FT: frozenset = frozenset({"Date", "Datetime", "Time"}) +_TEXT_FT: frozenset = frozenset({"Data", "Link", "Select", "Small Text", "Text", "Long Text"}) +_DATE_FT: frozenset = frozenset({"Date", "Datetime", "Time"}) def _field_type_compat(ft1: str, ft2: str) -> float: @@ -3121,22 +3400,24 @@ def smart_lookup_suggest( frappe.throw(_(f"You do not have read access to {target_doctype}")) import json + from rapidfuzz import fuzz try: import polars as pl + _USE_POLARS = True except ImportError: _USE_POLARS = False - src_headers = _fast_parse(source_headers) # [{fieldname, label, fieldtype, options}, …] + src_headers = _fast_parse(source_headers) # [{fieldname, label, fieldtype, options}, …] tgt_headers = _fast_parse(target_headers) - src_sample = _fast_parse(source_sample) # [[val, …], …] ≤ 200 rows - tgt_sample = _fast_parse(target_sample) + src_sample = _fast_parse(source_sample) # [[val, …], …] ≤ 200 rows + tgt_sample = _fast_parse(target_sample) - suggestions: list = [] - claimed_src: set = set() - claimed_tgt: set = set() + suggestions: list = [] + claimed_src: set = set() + claimed_tgt: set = set() # ── Layer 1: Frappe meta — Link fields pointing to target DocType ────────── if source_doctype and target_doctype: @@ -3145,15 +3426,17 @@ def smart_lookup_suggest( for df in src_meta.fields: if df.fieldtype == "Link" and df.options == target_doctype: if df.fieldname not in claimed_src: - suggestions.append({ - "source_col": df.fieldname, - "target_col": "name", - "strategy": "link_field", - "confidence": 0.97, - "reason": ( - f"'{df.label or df.fieldname}' is a Link field to {target_doctype}" - ), - }) + suggestions.append( + { + "source_col": df.fieldname, + "target_col": "name", + "strategy": "link_field", + "confidence": 0.97, + "reason": ( + f"'{df.label or df.fieldname}' is a Link field to {target_doctype}" + ), + } + ) claimed_src.add(df.fieldname) claimed_tgt.add("name") except Exception: @@ -3172,13 +3455,15 @@ def smart_lookup_suggest( continue if sf["fieldname"] == tf["fieldname"]: compat = _field_type_compat(sf.get("fieldtype", ""), tf.get("fieldtype", "")) - suggestions.append({ - "source_col": sf["fieldname"], - "target_col": tf["fieldname"], - "strategy": "header_match", - "confidence": round(0.98 * compat, 3), - "reason": f"Exact fieldname match: '{sf['fieldname']}'", - }) + suggestions.append( + { + "source_col": sf["fieldname"], + "target_col": tf["fieldname"], + "strategy": "header_match", + "confidence": round(0.98 * compat, 3), + "reason": f"Exact fieldname match: '{sf['fieldname']}'", + } + ) claimed_src_l2.add(sf["fieldname"]) claimed_tgt_l2.add(tf["fieldname"]) break @@ -3195,7 +3480,7 @@ def smart_lookup_suggest( # Works even when the report is filtered (low sample overlap) or when there's no meta. def _doctype_fk_bonus(si_idx, ti_idx): - """Return a 0–0.05 data-overlap bonus to add to structural base confidence.""" + """Return a 0-0.05 data-overlap bonus to add to structural base confidence.""" if not src_sample or not tgt_sample: return 0.0 src_v = {str(r[si_idx]) for r in src_sample if si_idx < len(r) and str(r[si_idx]).strip()} @@ -3207,10 +3492,8 @@ def _doctype_fk_bonus(si_idx, ti_idx): # 0c-A: source column named like target_doctype → FK to target.name if target_doctype and "name" not in claimed_tgt: - dt_snake = frappe.scrub(target_doctype) # "Customer" → "customer" - name_ti = next( - (i for i, tf in enumerate(tgt_headers) if tf["fieldname"] == "name"), None - ) + dt_snake = frappe.scrub(target_doctype) # "Customer" → "customer" + name_ti = next((i for i, tf in enumerate(tgt_headers) if tf["fieldname"] == "name"), None) if name_ti is not None: for si, sf in enumerate(src_headers): if sf["fieldname"] in claimed_src: @@ -3218,16 +3501,18 @@ def _doctype_fk_bonus(si_idx, ti_idx): fn = sf["fieldname"] if fn == dt_snake or fn == dt_snake + "_id": bonus = _doctype_fk_bonus(si, name_ti) - suggestions.append({ - "source_col": fn, - "target_col": "name", - "strategy": "primary_key_match", - "confidence": round(min(0.99, 0.94 + bonus), 3), - "reason": ( - f"'{sf.get('label') or fn}' column name matches DocType " - f"'{target_doctype}' — foreign key on {target_doctype}.name" - ), - }) + suggestions.append( + { + "source_col": fn, + "target_col": "name", + "strategy": "primary_key_match", + "confidence": round(min(0.99, 0.94 + bonus), 3), + "reason": ( + f"'{sf.get('label') or fn}' column name matches DocType " + f"'{target_doctype}' — foreign key on {target_doctype}.name" + ), + } + ) claimed_src.add(fn) claimed_tgt.add("name") break @@ -3235,9 +3520,7 @@ def _doctype_fk_bonus(si_idx, ti_idx): # 0c-B: source.name (primary key) → target column named like source_doctype if source_doctype and "name" not in claimed_src: dt_snake = frappe.scrub(source_doctype) - src_name_si = next( - (i for i, sf in enumerate(src_headers) if sf["fieldname"] == "name"), None - ) + src_name_si = next((i for i, sf in enumerate(src_headers) if sf["fieldname"] == "name"), None) if src_name_si is not None: for ti, tf in enumerate(tgt_headers): if tf["fieldname"] in claimed_tgt: @@ -3245,16 +3528,18 @@ def _doctype_fk_bonus(si_idx, ti_idx): fn = tf["fieldname"] if fn == dt_snake or fn == dt_snake + "_id": bonus = _doctype_fk_bonus(src_name_si, ti) - suggestions.append({ - "source_col": "name", - "target_col": fn, - "strategy": "primary_key_match", - "confidence": round(min(0.99, 0.94 + bonus), 3), - "reason": ( - f"'{tf.get('label') or fn}' column name matches source DocType " - f"'{source_doctype}' — join on {source_doctype}.name → {fn}" - ), - }) + suggestions.append( + { + "source_col": "name", + "target_col": fn, + "strategy": "primary_key_match", + "confidence": round(min(0.99, 0.94 + bonus), 3), + "reason": ( + f"'{tf.get('label') or fn}' column name matches source DocType " + f"'{source_doctype}' — join on {source_doctype}.name → {fn}" + ), + } + ) claimed_src.add("name") claimed_tgt.add(fn) break @@ -3268,14 +3553,10 @@ def _doctype_fk_bonus(si_idx, ti_idx): # Threshold lowered to 0.3: a filtered report may show only a subset of rows # but the join is still correct (e.g. 6/20 IGA customers appear in Customer sample). if src_sample and tgt_sample: - name_ti = next( - (i for i, tf in enumerate(tgt_headers) if tf["fieldname"] == "name"), None - ) + name_ti = next((i for i, tf in enumerate(tgt_headers) if tf["fieldname"] == "name"), None) if name_ti is not None and "name" not in claimed_tgt: tgt_names_raw = [ - str(r[name_ti]) - for r in tgt_sample - if name_ti < len(r) and str(r[name_ti]).strip() + str(r[name_ti]) for r in tgt_sample if name_ti < len(r) and str(r[name_ti]).strip() ] if tgt_names_raw: tgt_name_set = set(tgt_names_raw) @@ -3285,40 +3566,37 @@ def _doctype_fk_bonus(si_idx, ti_idx): for si, sf in enumerate(src_headers): if sf["fieldname"] in claimed_src: continue - src_raw = [ - str(r[si]) for r in src_sample if si < len(r) and str(r[si]).strip() - ] + src_raw = [str(r[si]) for r in src_sample if si < len(r) and str(r[si]).strip()] if not src_raw: continue if _USE_POLARS: - src_pl = pl.Series(src_raw).unique() + src_pl = pl.Series(src_raw).unique() overlap = int(src_pl.is_in(tgt_name_pl).sum()) - total = len(src_pl) + total = len(src_pl) else: src_set = set(src_raw) overlap = len(src_set & tgt_name_set) - total = len(src_set) + total = len(src_set) if total == 0: continue ratio = overlap / total if ratio >= 0.3: - src_set_py = set(src_raw) - union = len(src_set_py | tgt_name_set) - jaccard = overlap / union if union else 0.0 # Scale: 0.3 ratio→~0.82, 1.0 ratio→0.99 confidence = round(min(0.99, 0.72 + ratio * 0.27), 3) - suggestions.append({ - "source_col": sf["fieldname"], - "target_col": "name", - "strategy": "primary_key_match", - "confidence": confidence, - "reason": ( - f"{overlap}/{total} source values match target IDs " - f"({int(ratio * 100)}% hit-rate)" - ), - }) + suggestions.append( + { + "source_col": sf["fieldname"], + "target_col": "name", + "strategy": "primary_key_match", + "confidence": confidence, + "reason": ( + f"{overlap}/{total} source values match target IDs " + f"({int(ratio * 100)}% hit-rate)" + ), + } + ) claimed_src.add(sf["fieldname"]) claimed_tgt.add("name") break @@ -3327,9 +3605,7 @@ def _doctype_fk_bonus(si_idx, ti_idx): # Covers doctype→report joins where the report stores IDs in a non-`name` col # (e.g. Customer.name = "CUST-001" appears in Report.customer_name column). if src_sample and tgt_sample and "name" not in claimed_src: - src_name_si = next( - (i for i, sf in enumerate(src_headers) if sf["fieldname"] == "name"), None - ) + src_name_si = next((i for i, sf in enumerate(src_headers) if sf["fieldname"] == "name"), None) if src_name_si is not None: src_raw = [ str(r[src_name_si]) @@ -3345,39 +3621,29 @@ def _doctype_fk_bonus(si_idx, ti_idx): # pointing to the same target is fine — user picks which to apply. if tf["fieldname"] == "name": continue - tgt_raw = [ - str(r[ti]) - for r in tgt_sample - if ti < len(r) and str(r[ti]).strip() - ] + tgt_raw = [str(r[ti]) for r in tgt_sample if ti < len(r) and str(r[ti]).strip()] if not tgt_raw: continue tgt_set = set(tgt_raw) - inter = len(src_set & tgt_set) - ratio = inter / len(src_set) if src_set else 0.0 + inter = len(src_set & tgt_set) + ratio = inter / len(src_set) if src_set else 0.0 if ratio >= 0.3 and ratio > best_j: best_j, best_tf, best_cnt = ratio, tf, inter if best_tf: - best_ti_idx = next(i for i, tf in enumerate(tgt_headers) if tf is best_tf) - best_tgt_set = set( - str(r[best_ti_idx]) - for r in tgt_sample - if best_ti_idx < len(r) and str(r[best_ti_idx]).strip() + suggestions.append( + { + "source_col": "name", + "target_col": best_tf["fieldname"], + "strategy": "primary_key_match", + "confidence": round(min(0.96, 0.82 + best_j * 0.14), 3), + "reason": ( + f"{best_cnt}/{len(src_set)} source IDs match " + f"'{best_tf.get('label') or best_tf['fieldname']}' values " + f"({int(best_j * 100)}% hit-rate)" + ), + } ) - union = len(src_set | best_tgt_set) - jaccard = best_cnt / union if union else 0.0 - suggestions.append({ - "source_col": "name", - "target_col": best_tf["fieldname"], - "strategy": "primary_key_match", - "confidence": round(min(0.96, 0.82 + best_j * 0.14), 3), - "reason": ( - f"{best_cnt}/{len(src_set)} source IDs match " - f"'{best_tf.get('label') or best_tf['fieldname']}' values " - f"({int(best_j * 100)}% hit-rate)" - ), - }) claimed_src.add("name") claimed_tgt.add(best_tf["fieldname"]) @@ -3392,10 +3658,10 @@ def _doctype_fk_bonus(si_idx, ti_idx): for tf in tgt_headers: if tf["fieldname"] in claimed_tgt_l2: continue - src_label = (sf.get("label") or sf["fieldname"]).lower() - tgt_label = (tf.get("label") or tf["fieldname"]).lower() - lbl_score = fuzz.token_sort_ratio(src_label, tgt_label) - fn_score = fuzz.ratio( + src_label = (sf.get("label") or sf["fieldname"]).lower() + tgt_label = (tf.get("label") or tf["fieldname"]).lower() + lbl_score = fuzz.token_sort_ratio(src_label, tgt_label) + fn_score = fuzz.ratio( sf["fieldname"].replace("_", " "), tf["fieldname"].replace("_", " "), ) @@ -3404,17 +3670,19 @@ def _doctype_fk_bonus(si_idx, ti_idx): best_score, best_tf = score, tf if best_tf and best_score >= 75: compat = _field_type_compat(sf.get("fieldtype", ""), best_tf.get("fieldtype", "")) - suggestions.append({ - "source_col": sf["fieldname"], - "target_col": best_tf["fieldname"], - "strategy": "header_match", - "confidence": round((best_score / 100) * compat, 3), - "reason": ( - f"'{sf.get('label') or sf['fieldname']}' ≈ " - f"'{best_tf.get('label') or best_tf['fieldname']}' " - f"({best_score}% similarity)" - ), - }) + suggestions.append( + { + "source_col": sf["fieldname"], + "target_col": best_tf["fieldname"], + "strategy": "header_match", + "confidence": round((best_score / 100) * compat, 3), + "reason": ( + f"'{sf.get('label') or sf['fieldname']}' ≈ " + f"'{best_tf.get('label') or best_tf['fieldname']}' " + f"({best_score}% similarity)" + ), + } + ) claimed_src_l2.add(sf["fieldname"]) claimed_tgt_l2.add(best_tf["fieldname"]) @@ -3427,9 +3695,7 @@ def _doctype_fk_bonus(si_idx, ti_idx): for si, sf in enumerate(src_headers): if sf["fieldname"] in claimed_src: continue - src_raw = [ - str(r[si]) for r in src_sample if si < len(r) and str(r[si]).strip() - ] + src_raw = [str(r[si]) for r in src_sample if si < len(r) and str(r[si]).strip()] if not src_raw: continue @@ -3437,9 +3703,7 @@ def _doctype_fk_bonus(si_idx, ti_idx): for ti, tf in enumerate(tgt_headers): if tf["fieldname"] in claimed_tgt: continue - tgt_raw = [ - str(r[ti]) for r in tgt_sample if ti < len(r) and str(r[ti]).strip() - ] + tgt_raw = [str(r[ti]) for r in tgt_sample if ti < len(r) and str(r[ti]).strip()] if not tgt_raw: continue @@ -3452,22 +3716,23 @@ def _doctype_fk_bonus(si_idx, ti_idx): inter = len(src_set & tgt_set) union = len(src_set | tgt_set) - j = inter / union if union else 0.0 + j = inter / union if union else 0.0 if j > best_j: best_j, best_tf, best_cnt = j, tf, inter if best_tf and best_j >= 0.2: compat = _field_type_compat(sf.get("fieldtype", ""), best_tf.get("fieldtype", "")) - suggestions.append({ - "source_col": sf["fieldname"], - "target_col": best_tf["fieldname"], - "strategy": "data_content_match", - "confidence": round(best_j * compat, 3), - "reason": ( - f"{best_cnt} shared unique values " - f"({int(best_j * 100)}% Jaccard overlap)" - ), - }) + suggestions.append( + { + "source_col": sf["fieldname"], + "target_col": best_tf["fieldname"], + "strategy": "data_content_match", + "confidence": round(best_j * compat, 3), + "reason": ( + f"{best_cnt} shared unique values " f"({int(best_j * 100)}% Jaccard overlap)" + ), + } + ) claimed_src.add(sf["fieldname"]) claimed_tgt.add(best_tf["fieldname"]) @@ -3483,6 +3748,7 @@ def _doctype_fk_bonus(si_idx, ti_idx): # ── Schema graph & relational auto-expansion ────────────────────────────────── + @frappe.whitelist() def expand_relationship( source_doctype: str, @@ -3511,13 +3777,14 @@ def expand_relationship( _check_doctype_permission(source_doctype) _check_doctype_permission(target_doctype) - values = [str(v) for v in _fast_parse(source_values) if v] + values = [str(v) for v in _fast_parse(source_values) if v] fields_wanted = _fast_parse(return_fields) if not values: return {"columns": [], "rows": [], "cardinality": "unknown"} try: import polars as pl + _USE_POLARS = True except ImportError: _USE_POLARS = False @@ -3527,8 +3794,7 @@ def expand_relationship( # Auto-detect cardinality: does target have a Link field back to source? reverse_field = next( - (df.fieldname for df in tgt_meta.fields - if df.fieldtype == "Link" and df.options == source_doctype), + (df.fieldname for df in tgt_meta.fields if df.fieldtype == "Link" and df.options == source_doctype), None, ) cardinality = "1:N" if reverse_field else "N:1" @@ -3536,13 +3802,16 @@ def expand_relationship( # Auto-select return fields when not specified if not fields_wanted: fields_wanted = [ - df.fieldname for df in tgt_meta.fields + df.fieldname + for df in tgt_meta.fields if df.fieldtype in ("Currency", "Float", "Int", "Data", "Link", "Select") - and not df.hidden and getattr(df, "in_list_view", 0) + and not df.hidden + and getattr(df, "in_list_view", 0) ][:6] if not fields_wanted: fields_wanted = [ - df.fieldname for df in tgt_meta.fields + df.fieldname + for df in tgt_meta.fields if df.fieldtype in ("Currency", "Float", "Int") and not df.hidden ][:5] @@ -3552,8 +3821,10 @@ def expand_relationship( # ── 1:N aggregation presets ──────────────────────────────────────────── if cardinality == "1:N" and agg_preset in ("count", "sum", "average"): numeric_fields = [ - df.fieldname for df in tgt_meta.fields - if df.fieldtype in ("Currency", "Float", "Int") and not df.hidden + df.fieldname + for df in tgt_meta.fields + if df.fieldtype in ("Currency", "Float", "Int") + and not df.hidden and (not fields_wanted or df.fieldname in fields_wanted) ][:5] @@ -3568,16 +3839,16 @@ def expand_relationship( return { "columns": [ {"fieldname": "_src_key", "label": source_doctype}, - {"fieldname": "count", "label": f"Count of {target_doctype}"}, + {"fieldname": "count", "label": f"Count of {target_doctype}"}, ], - "rows": rows, + "rows": rows, "cardinality": cardinality, - "agg_preset": agg_preset, + "agg_preset": agg_preset, } if numeric_fields: pfx = "sum_" if agg_preset == "sum" else "avg_" - fn = "SUM" if agg_preset == "sum" else "AVG" + fn = "SUM" if agg_preset == "sum" else "AVG" agg_expr = ", ".join(f"{fn}(`{f}`) AS `{pfx}{f}`" for f in numeric_fields) sql = f""" SELECT `{reverse_field}` AS _src_key, {agg_expr} @@ -3589,17 +3860,18 @@ def expand_relationship( return { "columns": [ {"fieldname": "_src_key", "label": source_doctype}, - ] + [ + ] + + [ {"fieldname": f"{pfx}{f}", "label": f"{agg_preset.title()} {col_label.get(f, f)}"} for f in numeric_fields ], - "rows": rows, + "rows": rows, "cardinality": cardinality, - "agg_preset": agg_preset, + "agg_preset": agg_preset, } # ── N:1 or 1:N "latest" — row-level fetch ───────────────────────────── - fld_sql = ", ".join(f"`{f}`" for f in fields_wanted) if fields_wanted else "name" + fld_sql = ", ".join(f"`{f}`" for f in fields_wanted) if fields_wanted else "name" join_col = reverse_field if cardinality == "1:N" else "name" sql = f""" SELECT `{join_col}` AS _src_key, {fld_sql} @@ -3625,13 +3897,11 @@ def expand_relationship( rows = pl.from_dicts(rows).to_dicts() return { - "columns": [{"fieldname": "_src_key", "label": join_field}] + [ - {"fieldname": f, "label": col_label.get(f, f)} - for f in fields_wanted - ], - "rows": rows, + "columns": [{"fieldname": "_src_key", "label": join_field}] + + [{"fieldname": f, "label": col_label.get(f, f)} for f in fields_wanted], + "rows": rows, "cardinality": cardinality, - "agg_preset": agg_preset, + "agg_preset": agg_preset, } except Exception as e: @@ -3691,6 +3961,7 @@ def smart_lookup_fetch( # ── V3.3 — Activity column tag helpers ──────────────────────────────────────── + @frappe.whitelist() def get_doctype_tags(doctype): """Return all tags used on this DocType (distinct, sorted).""" @@ -3711,6 +3982,7 @@ def add_doc_tag(doctype, docname, tag): """Add a tag to a document.""" frappe.has_permission(doctype, "write", throw=True) from frappe.desk.doctype.tag.tag import add_tag + return add_tag(tag, doctype, docname) @@ -3719,6 +3991,7 @@ def remove_doc_tag(doctype, docname, tag): """Remove a tag from a document.""" frappe.has_permission(doctype, "write", throw=True) from frappe.desk.doctype.tag.tag import remove_tag + return remove_tag(tag, doctype, docname) @@ -3730,8 +4003,15 @@ def _get_active_formula_configs(): configs = frappe.get_all( "Excel Formula", filters={"is_active": 1, "is_system": 0}, - fields=["formula_name", "label", "category", "formula_type", - "source_doctype", "target_fieldname", "description"], + fields=[ + "formula_name", + "label", + "category", + "formula_type", + "source_doctype", + "target_fieldname", + "description", + ], order_by="formula_name asc", ) for cfg in configs: @@ -3746,11 +4026,28 @@ def _get_active_formula_configs(): def extend_bootinfo(bootinfo): """Inject active Excel Formula configs into frappe.boot so the JS plugin - can register dynamic HyperFormula functions before HOT initialises.""" + can register dynamic HyperFormula functions before HOT initialises. + Also ships the site-wide "Manual Save" switch (Excel View Settings).""" try: bootinfo.excel_formula_configs = _get_active_formula_configs() except Exception: bootinfo.excel_formula_configs = [] + bootinfo.excel_view_manual_save = _manual_save_enabled() + + +def _manual_save_enabled() -> int: + """Manual Save (Ctrl+S) is OFF unless an administrator switches it on in + Excel View Settings. Edits are auto-saved, as before, when the setting has never + been saved (a fresh install or an upgrade).""" + try: + stored = frappe.db.sql( + "select value from tabSingles where doctype=%s and field=%s", + ("Excel View Settings", "manual_save"), + ) + return int(stored[0][0] or 0) if stored else 0 + except Exception: + # The settings doctype does not exist yet (before the first migrate). + return 0 @frappe.whitelist() @@ -3884,6 +4181,7 @@ def seed_example_formulas(): # ── V3 IntelliFlow: DuckDB bulk fetch + schema endpoints ────────────────────── + @frappe.whitelist() def bulk_fetch_for_duckdb(doctype, filters="[]", limit=100000): """ @@ -3903,15 +4201,25 @@ def bulk_fetch_for_duckdb(doctype, filters="[]", limit=100000): parsed_filters = [] if filters: try: - raw = (_fast_parse(filters) if isinstance(filters, str) else (filters or [])) + raw = _fast_parse(filters) if isinstance(filters, str) else (filters or []) if isinstance(raw, list): parsed_filters = raw except Exception: pass meta = frappe.get_meta(doctype) - SKIP_TYPES = {"Column Break", "Section Break", "Tab Break", "Fold", - "Heading", "HTML", "Custom HTML", "Table", "Table MultiSelect", "Password"} + SKIP_TYPES = { + "Column Break", + "Section Break", + "Tab Break", + "Fold", + "Heading", + "HTML", + "Custom HTML", + "Table", + "Table MultiSelect", + "Password", + } fields = ["name", "creation", "modified", "owner"] for df in meta.fields: if df.fieldtype in SKIP_TYPES or df.is_virtual: @@ -3937,43 +4245,43 @@ def bulk_fetch_for_duckdb(doctype, filters="[]", limit=100000): # ── Arrow IPC binary transport (3-5x smaller + zero CSV re-encoding) ── try: - import pyarrow as pa import base64 + import pyarrow as pa + if rows: # Build columnar arrays — cast everything to string to avoid type # inference issues across Frappe field types (dates, decimals, etc.) col_arrays = {} for f in unique_fields: col_arrays[f] = pa.array( - [str(r[f]) if r[f] is not None else "" for r in rows], - type=pa.string() + [str(r[f]) if r[f] is not None else "" for r in rows], type=pa.string() ) table = pa.table(col_arrays) else: schema = pa.schema([(f, pa.string()) for f in unique_fields]) - table = pa.table({f: pa.array([], type=pa.string()) for f in unique_fields}, schema=schema) + table = pa.table({f: pa.array([], type=pa.string()) for f in unique_fields}, schema=schema) - sink = pa.BufferOutputStream() + sink = pa.BufferOutputStream() writer = pa.ipc.new_stream(sink, table.schema) writer.write_table(table) writer.close() arrow_b64 = base64.b64encode(sink.getvalue().to_pybytes()).decode("ascii") return { - "doctype": doctype, - "fields": unique_fields, - "count": len(rows), - "arrow_ipc": arrow_b64, # base64 Arrow IPC stream + "doctype": doctype, + "fields": unique_fields, + "count": len(rows), + "arrow_ipc": arrow_b64, # base64 Arrow IPC stream } except Exception: # Fallback: plain JSON rows (always works) return { "doctype": doctype, - "rows": rows, - "count": len(rows), - "fields": unique_fields, + "rows": rows, + "count": len(rows), + "fields": unique_fields, } @@ -3988,15 +4296,23 @@ def get_doctype_schema(doctypes): """ if isinstance(doctypes, str): try: - doctypes = (_fast_parse(doctypes) if isinstance(doctypes, str) else (doctypes or [])) + doctypes = _fast_parse(doctypes) if isinstance(doctypes, str) else (doctypes or []) except Exception: doctypes = [doctypes] if not isinstance(doctypes, list): doctypes = [doctypes] - SKIP_TYPES = {"Column Break", "Section Break", "Tab Break", "Fold", - "Heading", "HTML", "Custom HTML", "Password"} + SKIP_TYPES = { + "Column Break", + "Section Break", + "Tab Break", + "Fold", + "Heading", + "HTML", + "Custom HTML", + "Password", + } result = {} for doctype in doctypes: @@ -4011,26 +4327,30 @@ def get_doctype_schema(doctypes): fields = [] # Always include name first - fields.append({ - "fieldname": "name", - "label": "Name (ID)", - "fieldtype": "Data", - "options": "", - "reqd": 1, - }) + fields.append( + { + "fieldname": "name", + "label": "Name (ID)", + "fieldtype": "Data", + "options": "", + "reqd": 1, + } + ) for df in meta.fields: if df.fieldtype in SKIP_TYPES: continue if df.is_virtual: continue - fields.append({ - "fieldname": df.fieldname, - "label": df.label or df.fieldname, - "fieldtype": df.fieldtype, - "options": df.options or "", - "reqd": int(bool(df.reqd)), - }) + fields.append( + { + "fieldname": df.fieldname, + "label": df.label or df.fieldname, + "fieldtype": df.fieldtype, + "options": df.options or "", + "reqd": int(bool(df.reqd)), + } + ) result[doctype] = { "module": meta.module or "", @@ -4051,19 +4371,28 @@ def get_doctype_schema(doctypes): # so cache invalidation, validation, and Custom DocPerm copy-on-write all happen # exactly as they do in Frappe's own Permission Manager page. -_PERM_BLOCKED_DOCTYPES = frozenset({ - "DocType", "DocField", "DocPerm", "Custom DocPerm", - "Property Setter", "Role", "User", "Session", - "DefaultValue", "Error Log", "Access Log", -}) +_PERM_BLOCKED_DOCTYPES = frozenset( + { + "DocType", + "DocField", + "DocPerm", + "Custom DocPerm", + "Property Setter", + "Role", + "User", + "Session", + "DefaultValue", + "Error Log", + "Access Log", + } +) def _perm_guard(doctype): """Block meta-system doctypes. Role check is handled by @excel_whitelist.""" if doctype in _PERM_BLOCKED_DOCTYPES: frappe.throw( - f"Permissions for system DocType '{frappe.bold(doctype)}' cannot be " - "managed from Excel View.", + f"Permissions for system DocType '{frappe.bold(doctype)}' cannot be " "managed from Excel View.", frappe.PermissionError, ) @@ -4077,6 +4406,7 @@ def get_doctype_permissions(doctype): """ _perm_guard(doctype) from frappe.core.page.permission_manager import permission_manager as pm + return pm.get_permissions(doctype=doctype) @@ -4090,6 +4420,7 @@ def update_role_permission(doctype, role, permlevel, ptype, value=None, if_owner """ _perm_guard(doctype) from frappe.core.page.permission_manager import permission_manager as pm + return pm.update( doctype=doctype, role=role, @@ -4109,6 +4440,7 @@ def add_role_permission(doctype, role, permlevel=0): """ _perm_guard(doctype) from frappe.permissions import add_permission + add_permission(doctype, role, frappe.utils.cint(permlevel)) @@ -4121,6 +4453,7 @@ def remove_role_permission(doctype, role, permlevel, if_owner=0): """ _perm_guard(doctype) from frappe.core.page.permission_manager import permission_manager as pm + pm.remove( doctype=doctype, role=role, @@ -4138,6 +4471,7 @@ def reset_doctype_permissions(doctype): """ _perm_guard(doctype) from frappe.core.page.permission_manager import permission_manager as pm + pm.reset(doctype=doctype) @@ -4149,16 +4483,25 @@ def get_field_permlevels(doctype): is always the live value — no separate Property Setter query needed. """ _perm_guard(doctype) - SKIP_TYPES = frozenset({ - "Section Break", "Column Break", "Tab Break", - "Fold", "Heading", "HTML", "Custom HTML", "Password", - "Table", "Table MultiSelect", - }) + SKIP_TYPES = frozenset( + { + "Section Break", + "Column Break", + "Tab Break", + "Fold", + "Heading", + "HTML", + "Custom HTML", + "Password", + "Table", + "Table MultiSelect", + } + ) meta = frappe.get_meta(doctype) return [ { "fieldname": df.fieldname, - "label": df.label or df.fieldname, + "label": df.label or df.fieldname, "fieldtype": df.fieldtype, "permlevel": frappe.utils.cint(df.permlevel), } @@ -4184,14 +4527,17 @@ def set_field_permlevel(doctype, fieldname, permlevel): if not meta.get_field(fieldname): frappe.throw(f"Field '{frappe.bold(fieldname)}' not found in {frappe.bold(doctype)}.") - frappe.make_property_setter({ - "doctype": doctype, - "doctype_or_field": "DocField", - "fieldname": fieldname, - "property": "permlevel", - "value": permlevel, - "property_type": "Int", - }, is_system_generated=False) + frappe.make_property_setter( + { + "doctype": doctype, + "doctype_or_field": "DocField", + "fieldname": fieldname, + "property": "permlevel", + "value": permlevel, + "property_type": "Int", + }, + is_system_generated=False, + ) # Belt-and-suspenders: schedule one more clear after commit frappe.db.after_commit.add(lambda: frappe.clear_cache(doctype=doctype)) @@ -4204,10 +4550,7 @@ def get_role_user_counts(): Single aggregating query — O(1) round-trips regardless of role count. """ rows = frappe.db.sql( - "SELECT role, COUNT(*) AS cnt" - " FROM `tabHas Role`" - " WHERE parenttype = 'User'" - " GROUP BY role", + "SELECT role, COUNT(*) AS cnt" " FROM `tabHas Role`" " WHERE parenttype = 'User'" " GROUP BY role", as_dict=True, ) return {r.role: r.cnt for r in rows} @@ -4221,6 +4564,7 @@ def get_all_roles(): """ try: from frappe.permissions import AUTOMATIC_ROLES + restricted = set(AUTOMATIC_ROLES) except ImportError: restricted = {"All", "Guest"} @@ -4278,9 +4622,9 @@ def get_access_profiles(): role_profiles = [ { - "name": rp.name, + "name": rp.name, "user_count": rp_counts.get(rp.name, 0), - "roles": rp_role_map.get(rp.name, []), + "roles": rp_role_map.get(rp.name, []), } for rp in rp_rows ] @@ -4312,8 +4656,8 @@ def get_access_profiles(): module_profiles = [ { - "name": mp.name, - "user_count": mp_counts.get(mp.name, 0), + "name": mp.name, + "user_count": mp_counts.get(mp.name, 0), "blocked_modules": mp_mod_map.get(mp.name, []), } for mp in mp_rows @@ -4321,16 +4665,13 @@ def get_access_profiles(): # ── All installed modules ────────────────────────────────────────────── from frappe.config import get_modules_from_all_apps - all_modules = sorted( - m["module_name"] - for m in get_modules_from_all_apps() - if m.get("module_name") - ) + + all_modules = sorted(m["module_name"] for m in get_modules_from_all_apps() if m.get("module_name")) return { - "role_profiles": role_profiles, + "role_profiles": role_profiles, "module_profiles": module_profiles, - "all_modules": all_modules, + "all_modules": all_modules, } @@ -4342,6 +4683,7 @@ def save_role_profile(profile_name, roles): roles: JSON-encoded list of role name strings. """ import json as _json + role_list = _fast_parse(roles) if isinstance(roles, str) else list(roles) doc = frappe.get_doc("Role Profile", profile_name) @@ -4359,6 +4701,7 @@ def save_module_profile(profile_name, blocked_modules): blocked_modules: JSON-encoded list of module name strings. """ import json as _json + mod_list = _fast_parse(blocked_modules) if isinstance(blocked_modules, str) else list(blocked_modules) doc = frappe.get_doc("Module Profile", profile_name) diff --git a/excel_view/excel_view/doctype/excel_view_settings/__init__.py b/excel_view/excel_view/doctype/excel_view_settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.json b/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.json new file mode 100644 index 0000000..3b7145b --- /dev/null +++ b/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.json @@ -0,0 +1,41 @@ +{ + "actions": [], + "allow_rename": 0, + "creation": "2026-10-07 00:00:00.000000", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "manual_save" + ], + "fields": [ + { + "default": "0", + "description": "When ticked, edits in the Excel View grid are NOT saved automatically. They wait until the user presses Ctrl+S (or leaves the page). When unticked (the default), edits are saved about one second after each change.", + "fieldname": "manual_save", + "fieldtype": "Check", + "label": "Manual Save (Ctrl+S)" + } + ], + "index_web_pages_for_search": 0, + "issingle": 1, + "links": [], + "modified": "2026-10-07 00:00:00.000000", + "modified_by": "Administrator", + "module": "Excel View", + "name": "Excel View Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.py b/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.py new file mode 100644 index 0000000..a874a96 --- /dev/null +++ b/excel_view/excel_view/doctype/excel_view_settings/excel_view_settings.py @@ -0,0 +1,8 @@ +# Copyright (c) 2026, Ujjwal Aggrawal and contributors +# For license information, please see license.txt + +from frappe.model.document import Document + + +class ExcelViewSettings(Document): + pass diff --git a/excel_view/public/js/excel/components/context_menu.js b/excel_view/public/js/excel/components/context_menu.js index d815388..9b8996e 100644 --- a/excel_view/public/js/excel/components/context_menu.js +++ b/excel_view/public/js/excel/components/context_menu.js @@ -47,19 +47,14 @@ frappe.views.excel.ContextMenu = class ContextMenu { remove_row: { name: () => { const sel = board.hot.getSelected(); - const count = - sel && sel.length ? Math.abs(sel[0][2] - sel[0][0]) + 1 : 1; - return count > 1 - ? __("Remove {0} rows", [count]) - : __("Remove row"); + const count = sel && sel.length ? Math.abs(sel[0][2] - sel[0][0]) + 1 : 1; + return count > 1 ? __("Remove {0} rows", [count]) : __("Remove row"); }, callback: (key, selection) => { const start = Math.min(selection[0].start.row, selection[0].end.row); const end = Math.max(selection[0].start.row, selection[0].end.row); frappe.confirm( - __("Delete {0} record(s)? This cannot be undone.", [ - end - start + 1, - ]), + __("Delete {0} record(s)? This cannot be undone.", [end - start + 1]), () => board.data_manager.delete_rows(start, end) ); }, @@ -67,30 +62,30 @@ frappe.views.excel.ContextMenu = class ContextMenu { }, // V3.1 — Hide / Unhide rows (HOT 6: uses board._hide_rows / _unhide_all_rows) - hide_row: { - name: () => { - const sel = board.hot.getSelected(); - const count = sel?.length ? Math.abs(sel[0][2] - sel[0][0]) + 1 : 1; - return count > 1 ? __("Hide {0} Rows", [count]) : __("Hide Row"); - }, - callback: (key, selection) => { - const rows = []; - selection.forEach(({ start, end }) => { - const r1 = Math.min(start.row, end.row); - const r2 = Math.max(start.row, end.row); - for (let r = r1; r <= r2; r++) rows.push(r); - }); - board._hide_rows(rows); + hide_row: { + name: () => { + const sel = board.hot.getSelected(); + const count = sel?.length ? Math.abs(sel[0][2] - sel[0][0]) + 1 : 1; + return count > 1 ? __("Hide {0} Rows", [count]) : __("Hide Row"); + }, + callback: (key, selection) => { + const rows = []; + selection.forEach(({ start, end }) => { + const r1 = Math.min(start.row, end.row); + const r2 = Math.max(start.row, end.row); + for (let r = r1; r <= r2; r++) rows.push(r); + }); + board._hide_rows(rows); + }, }, - }, - show_rows: { - name: () => __("Unhide Rows"), - disabled: () => !board._hidden_rows?.length, - callback: () => board._unhide_all_rows(), - }, + show_rows: { + name: () => __("Unhide Rows"), + disabled: () => !board._hidden_rows?.length, + callback: () => board._unhide_all_rows(), + }, - sep1: "---------", + sep1: "---------", // ── Clipboard ───────────────────────────────────────────────── copy: { @@ -167,7 +162,11 @@ frappe.views.excel.ContextMenu = class ContextMenu { // Collect unique column indices from all HOT selection ranges const cols = new Set(); selection.forEach(({ start, end }) => { - for (let c = Math.min(start.col, end.col); c <= Math.max(start.col, end.col); c++) { + for ( + let c = Math.min(start.col, end.col); + c <= Math.max(start.col, end.col); + c++ + ) { cols.add(c); } }); @@ -193,10 +192,7 @@ frappe.views.excel.ContextMenu = class ContextMenu { return __("Freeze up to column: {0}", [label]); }, callback: (key, selection) => { - const col = Math.max( - selection[0].start.col, - selection[0].end.col - ); + const col = Math.max(selection[0].start.col, selection[0].end.col); board._set_freeze(col + 1); }, }, @@ -222,14 +218,14 @@ frappe.views.excel.ContextMenu = class ContextMenu { name: () => __("Remove Blank Column"), callback: (key, selection) => { const start_col = Math.min(selection[0].start.col, selection[0].end.col); - const end_col = Math.max(selection[0].start.col, selection[0].end.col); + const end_col = Math.max(selection[0].start.col, selection[0].end.col); board._remove_blank_columns(start_col, end_col); }, disabled: () => { const sel = board.hot.getSelected(); if (!sel?.length) return true; const start_col = Math.min(sel[0][1], sel[0][3]); - const end_col = Math.max(sel[0][1], sel[0][3]); + const end_col = Math.max(sel[0][1], sel[0][3]); for (let c = start_col; c <= end_col; c++) { if (board.columns[c]?._is_blank_col) return false; } @@ -249,7 +245,7 @@ frappe.views.excel.ContextMenu = class ContextMenu { const sel = board.hot.getSelected(); if (!sel?.length) return __("Remove formula column"); const start_col = Math.min(sel[0][1], sel[0][3]); - const end_col = Math.max(sel[0][1], sel[0][3]); + const end_col = Math.max(sel[0][1], sel[0][3]); let count = 0; for (let c = start_col; c <= end_col; c++) { if (board.columns[c]?._is_formula_col) count++; @@ -259,21 +255,15 @@ frappe.views.excel.ContextMenu = class ContextMenu { : __("Remove formula column"); }, callback: (key, selection) => { - const start_col = Math.min( - selection[0].start.col, - selection[0].end.col - ); - const end_col = Math.max( - selection[0].start.col, - selection[0].end.col - ); + const start_col = Math.min(selection[0].start.col, selection[0].end.col); + const end_col = Math.max(selection[0].start.col, selection[0].end.col); board._remove_formula_columns(start_col, end_col); }, disabled: () => { const sel = board.hot.getSelected(); if (!sel?.length) return true; const start_col = Math.min(sel[0][1], sel[0][3]); - const end_col = Math.max(sel[0][1], sel[0][3]); + const end_col = Math.max(sel[0][1], sel[0][3]); for (let c = start_col; c <= end_col; c++) { if (board.columns[c]?._is_formula_col) return false; } @@ -281,7 +271,6 @@ frappe.views.excel.ContextMenu = class ContextMenu { }, }, - // ── Fill Column ─────────────────────────────────────────────── fill_column: { name: () => __("Fill Column ↓ (All Rows)"), @@ -301,6 +290,38 @@ frappe.views.excel.ContextMenu = class ContextMenu { sep5: "---------", + // ── Filter by this value ────────────────────────────────────── + // Adds a filter on the clicked cell's column and value (an empty cell + // filters for "not set"). Uses the same filter bar as the funnel. + filter_by_value: { + name: () => __("Filter by this value"), + disabled: () => { + const sel = board.hot.getSelectedLast(); + const col = sel && board.columns[Math.min(sel[1], sel[3])]; + return ( + !col || + /^_/.test(col.data) || + !!col._is_formula_col || + !!col._is_blank_col || + !!col._is_join_col + ); + }, + callback: () => { + const sel = board.hot.getSelectedLast(); + if (!sel) return; + const col = board.columns[Math.min(sel[1], sel[3])]; + const rec = board.list_view.data[Math.min(sel[0], sel[2])]; + if (!col || !rec || /^_/.test(col.data)) return; + const v = rec[col.data]; + const empty = v === null || v === undefined || v === ""; + board.list_view.filter_area.add([ + [board.doctype, col.data, empty ? "is" : "=", empty ? "not set" : v], + ]); + }, + }, + + sep5b: "---------", + // ── Open form ───────────────────────────────────────────────── open_form: { name: () => __("Open in Form View"), diff --git a/excel_view/public/js/excel/components/data_manager.js b/excel_view/public/js/excel/components/data_manager.js index b3d70d7..fe4a270 100644 --- a/excel_view/public/js/excel/components/data_manager.js +++ b/excel_view/public/js/excel/components/data_manager.js @@ -19,10 +19,7 @@ frappe.views.excel.DataManager = class DataManager { this.board = opts.board; // { doc_name: { fieldname: value, ... }, ... } this._save_queue = {}; - this._save_debounced = frappe.utils.debounce( - this._flush_saves.bind(this), - 800 - ); + this._save_debounced = frappe.utils.debounce(this._flush_saves.bind(this), 800); this._dirty = false; } @@ -88,8 +85,16 @@ frappe.views.excel.DataManager = class DataManager { const col_def = columns.find((c) => c.data === fieldname); // Skip formula/join columns — they exist only in the grid, never in the DB // Skip _is_new rows — they are pending inline inserts, not yet in the DB - if (!doc || doc._is_new || !col_def || col_def._readonly || col_def._is_name_col - || col_def._is_formula_col || col_def._is_join_col) return; + if ( + !doc || + doc._is_new || + !col_def || + col_def._readonly || + col_def._is_name_col || + col_def._is_formula_col || + col_def._is_join_col + ) + return; const doc_name = doc.name; if (!doc_name || !fieldname) return; @@ -105,7 +110,9 @@ frappe.views.excel.DataManager = class DataManager { this._dirty = true; this._show_dirty_indicator(); - this._save_debounced(); + // Manual-save mode: edits wait until Ctrl+S (or leaving the page). Otherwise + // they are saved about a second after the last edit. + if (!this.board.manual_save) this._save_debounced(); } /** @@ -131,21 +138,38 @@ frappe.views.excel.DataManager = class DataManager { error: () => {}, }); const errors = r.message?.errors || []; + this._mark_failed_cells(errors, updates); if (errors.length) { - // Show the first error in the friendly dialog; toast a count if multiple + // Show the first error in the friendly dialog; toast a count if multiple. + // Every failed cell is also marked red (see _mark_failed_cells). const first = errors[0]; frappe.views.excel.show_error( - { _server_messages: JSON.stringify([JSON.stringify({ message: first.error })]) }, + { + _server_messages: JSON.stringify([ + JSON.stringify({ message: first.error }), + ]), + }, errors.length > 1 ? __("{0} record(s) could not be saved", [errors.length]) : __("Saving {0}", [first.name]) ); - } else { - this._dirty = false; - this._clear_dirty_indicator(); - frappe.views.excel.toast(__("Saved"), "success", 2000); } + // Refused rows were reverted by the reload, so they are no longer waiting. + // Only edits typed while this request was running are still unsaved. + if (!errors.length || !Object.keys(this._save_queue).length) { + this._dirty = Object.keys(this._save_queue).length > 0; + if (!this._dirty) this._clear_dirty_indicator(); + } + if (!errors.length) frappe.views.excel.toast(__("Saved"), "success", 2000); } catch (err) { + // The whole request failed (session expired, network down, no permission). Nothing + // was stored, so put the edits back: Ctrl+S retries them and the leave warning + // stays on. Edits typed since the request went out win over the older ones. + Object.entries(queue).forEach(([name, fields]) => { + this._save_queue[name] = { ...fields, ...(this._save_queue[name] || {}) }; + }); + this._dirty = true; + this._show_dirty_indicator(); frappe.views.excel.show_error(err, __("Saving changes to {0}", [doctype])); } } @@ -182,10 +206,7 @@ frappe.views.excel.DataManager = class DataManager { } else { frappe.show_alert( { - message: __( - "{0} record(s) deleted", - [to_delete.length] - ), + message: __("{0} record(s) deleted", [to_delete.length]), indicator: "green", }, 2 @@ -212,4 +233,39 @@ frappe.views.excel.DataManager = class DataManager { is_dirty() { return this._dirty || Object.keys(this._save_queue).length > 0; } + + // ── Failed rows ─────────────────────────────────────────────────────────── + + /** + * Colour every cell the server refused, with the server's message as a tooltip. + * The server returns one entry per failed row: {name, error, fields: [fieldnames]}. + * A failed cell stays marked until the user edits it again. + */ + _mark_failed_cells(errors, updates) { + const board = this.board; + if (!board._failed_cells) board._failed_cells = new Map(); + const plain = (s) => + String(s || "") + .replace(/<[^>]*>/g, "") + .trim(); + + // A successful save of a cell clears an older mark on it. + const failed_rows = new Set(errors.map((e) => e.name)); + updates.forEach((u) => { + if (failed_rows.has(u.name)) return; + Object.keys(u.fields).forEach((f) => board._failed_cells.delete(`${u.name}|${f}`)); + }); + + errors.forEach((e) => { + const upd = updates.find((u) => u.name === e.name); + const fields = e.fields || Object.keys(upd?.fields || {}); + fields.forEach((f) => board._failed_cells.set(`${e.name}|${f}`, plain(e.error))); + }); + board.hot?.render(); + + if (errors.length) { + // The server did not store these edits: reload so the cells show what is saved. + setTimeout(() => board.list_view?.refresh(), 300); + } + } }; diff --git a/excel_view/public/js/excel/components/excel_board.js b/excel_view/public/js/excel/components/excel_board.js index 4174356..096404c 100644 --- a/excel_view/public/js/excel/components/excel_board.js +++ b/excel_view/public/js/excel/components/excel_board.js @@ -34,6 +34,9 @@ frappe.views.ExcelBoard = class ExcelBoard { this.columns = []; // Sparse map of per-cell formatting: { "row:col": { bold, italic, ... } } this.format_store = {}; + // Site-wide "Manual Save" switch (Excel View Settings). When on, edits wait until + // Ctrl+S (or leaving the page) instead of being saved about 1 second after Enter. + this.manual_save = !!frappe.boot.excel_view_manual_save; // Snapshot of list_view.fields at board creation time so _deselect() can // restore it after a workbook (which overwrites list_view.fields) is closed. this._default_list_view_fields = [...(this.list_view.fields || [])]; @@ -119,12 +122,12 @@ frappe.views.ExcelBoard = class ExcelBoard { this.workbook_manager = new frappe.views.excel.WorkbookManager({ board: this }); // V2.6 — Chart / CF / Pivot managers (toolbar delegates to these) - this.cf_manager = new frappe.views.excel.CFManager({ board: this }); - this.chart_manager = new frappe.views.excel.ChartManager({ board: this }); - this.pivot_builder = new frappe.views.excel.PivotBuilder({ board: this }); - this.dashboard_manager = new frappe.views.excel.DashboardManager({ board: this }); + this.cf_manager = new frappe.views.excel.CFManager({ board: this }); + this.chart_manager = new frappe.views.excel.ChartManager({ board: this }); + this.pivot_builder = new frappe.views.excel.PivotBuilder({ board: this }); + this.dashboard_manager = new frappe.views.excel.DashboardManager({ board: this }); // V3.5 — Inline child table expansion - this.child_table_manager = new frappe.views.excel.ChildTableManager({ board: this }); + this.child_table_manager = new frappe.views.excel.ChildTableManager({ board: this }); // 2. Load persisted freeze state (needed before _init_hot) this._frozen_cols = this.column_manager.load_freeze(); @@ -137,7 +140,7 @@ frappe.views.ExcelBoard = class ExcelBoard { if (this._ct_fieldnames.length) { this.column_manager.fields = [ ...this.column_manager.fields, - ...this._ct_fieldnames.map(f => [f, this.doctype]), + ...this._ct_fieldnames.map((f) => [f, this.doctype]), ]; } @@ -186,7 +189,7 @@ frappe.views.ExcelBoard = class ExcelBoard { // Restore hidden columns AFTER HOT init so _sync_visible_columns can call updateSettings const _saved_hidden_cols = frappe.get_user_settings(this.doctype)?.excel_hidden_cols; if (Array.isArray(_saved_hidden_cols) && _saved_hidden_cols.length) { - _saved_hidden_cols.forEach(key => this._hidden_col_keys.add(key)); + _saved_hidden_cols.forEach((key) => this._hidden_col_keys.add(key)); this._sync_visible_columns(); } // Restore Smart Lookup configs (column defs only — data re-joined after first refresh) @@ -196,7 +199,7 @@ frappe.views.ExcelBoard = class ExcelBoard { } // V3.1 — Restore manual row heights AFTER HOT init (plugin must exist) const _saved_rh = frappe.get_user_settings(this.doctype)?.excel_row_heights; - if (Array.isArray(_saved_rh) && _saved_rh.some(h => h != null)) { + if (Array.isArray(_saved_rh) && _saved_rh.some((h) => h != null)) { setTimeout(() => { const rh_plugin = this.hot?.getPlugin("manualRowResize"); if (rh_plugin) { @@ -253,7 +256,8 @@ frappe.views.ExcelBoard = class ExcelBoard { const saved_gridlines = frappe.get_user_settings(this.doctype)?.excel_hide_gridlines; if (saved_gridlines) this.$hot_container?.addClass("ev-hide-gridlines"); - const saved_freeze_rows = frappe.get_user_settings(this.doctype)?.excel_view_freeze_rows || 0; + const saved_freeze_rows = + frappe.get_user_settings(this.doctype)?.excel_view_freeze_rows || 0; this._frozen_rows = saved_freeze_rows; if (saved_freeze_rows > 0) { setTimeout(() => this.hot?.updateSettings({ fixedRowsTop: saved_freeze_rows }), 0); @@ -268,7 +272,9 @@ frappe.views.ExcelBoard = class ExcelBoard { // Restore formula column templates from user_settings (no workbook needed). // Deduplicate by key — a past bug could produce two entries with the same // __fml_N__ key; keep only the last occurrence (most recently saved). - const saved_formula_col_templates = frappe.get_user_settings(this.doctype)?.excel_formula_col_templates; + const saved_formula_col_templates = frappe.get_user_settings( + this.doctype + )?.excel_formula_col_templates; if (Array.isArray(saved_formula_col_templates) && saved_formula_col_templates.length) { const seen_keys = new Map(); for (const fc of saved_formula_col_templates) { @@ -278,7 +284,9 @@ frappe.views.ExcelBoard = class ExcelBoard { } // Restore blank column configs from user_settings - const saved_blank_col_configs = frappe.get_user_settings(this.doctype)?.excel_blank_col_configs; + const saved_blank_col_configs = frappe.get_user_settings( + this.doctype + )?.excel_blank_col_configs; if (Array.isArray(saved_blank_col_configs) && saved_blank_col_configs.length) { this._pending_blank_col_configs = saved_blank_col_configs; } @@ -340,14 +348,18 @@ frappe.views.ExcelBoard = class ExcelBoard { // position:relative is required so the child-table panel (position:absolute) // is positioned relative to this container, not the viewport. - this.$hot_container = $('
').appendTo(this.$grid_main); + this.$hot_container = $( + '
' + ).appendTo(this.$grid_main); // Dashboard canvas — shown when a Dashboard sheet is active, hidden otherwise this.$dashboard_canvas = $('
').appendTo(this.$grid_main); this.$dashboard_canvas.hide(); // Status bar — fixed footer below the grid - this.$status_bar_container = $('
').appendTo(this.$grid_main); + this.$status_bar_container = $('
').appendTo( + this.$grid_main + ); // Right sidebar slot — used by Smart Lookup, Agent Mode, etc. // Width transitions 0 → 300px; grid_main shrinks automatically (flex). @@ -405,7 +417,7 @@ frappe.views.ExcelBoard = class ExcelBoard { // V3.5 — CT spacer row: height = panel height so rows below are pushed down const spacer_h = this.child_table_manager?.spacer_height_for_row(row); if (spacer_h != null) return spacer_h; - return this.columns?.some(c => c._is_meta_col || c._is_social_col) ? 42 : 23; + return this.columns?.some((c) => c._is_meta_col || c._is_social_col) ? 42 : 23; }, columnSorting: true, allowInsertRow: this.list_view.can_create, @@ -420,7 +432,7 @@ frappe.views.ExcelBoard = class ExcelBoard { outsideClickDeselects: false, // Column sort via dropdown — full autoFilter disabled; Frappe's sidebar - // handles filtering so we expose only sort_asc / sort_desc in the header menu. + // handles filtering so we expose only sort_asc / sort_desc in the header menu. filters: false, dropdownMenu: ["sort_asc", "sort_desc"], @@ -430,7 +442,6 @@ frappe.views.ExcelBoard = class ExcelBoard { // V2.6 — Merge cells plugin enabled mergeCells: true, - // Frozen columns — restored from user_settings fixedColumnsLeft: this._frozen_cols, @@ -463,7 +474,9 @@ frappe.views.ExcelBoard = class ExcelBoard { if (d._is_anomaly) { meta.className = ((meta.className || "") + " ev-anomaly-row").trim(); } else if (d._cluster !== undefined && d._cluster !== null) { - meta.className = ((meta.className || "") + ` ev-cluster-${d._cluster % 6}`).trim(); + meta.className = ( + (meta.className || "") + ` ev-cluster-${d._cluster % 6}` + ).trim(); } // Inline insert: make all editable columns writable for the pending new row if (d._is_new && row === this._new_row_idx) { @@ -497,7 +510,11 @@ frappe.views.ExcelBoard = class ExcelBoard { }, afterRowResize: (row, size) => this._on_row_resize(row, size), afterRender: () => this._on_render(), - afterRenderer: (TD, row, col, prop, value) => this._apply_cell_format(TD, row, col, value), + afterRenderer: (TD, row, col, prop, value) => { + this._apply_cell_format(TD, row, col, value); + // Runs last, so no early return inside _apply_cell_format can overwrite it. + this._mark_failed_cell(TD, row, col); + }, afterGetColHeader: (col, TH) => this._apply_col_header_format(TH, col), afterOnCellMouseDown: (e, coords) => { this._on_tree_row_click(e, coords); @@ -555,24 +572,30 @@ frappe.views.ExcelBoard = class ExcelBoard { const row_data = this.list_view.data?.[row]; if (row_data?._tree_is_header && row_data._tree_size > 1) { const expanded = this._expanded_keys?.has(row_data._tree_group_key); - TH.innerHTML = `
${expanded ? '▼' : '▶'} ${row_data._tree_size}
`; + TH.innerHTML = `
${expanded ? "▼" : "▶"} ${ + row_data._tree_size + }
`; } else if (row_data?._tree_is_child) { TH.innerHTML = `
└
`; } }, // V3.1 — intercept F4; V3.3 — intercept Ctrl+E (Flash Fill); V3.3 — Ctrl+D (Fill Down) + // e.key is lower-cased so Ctrl+E / Ctrl+D also work with Caps Lock on. beforeKeyDown: (e) => { + const key = (e.key || "").toLowerCase(); if (e.key === "F4" && !e.ctrlKey && !e.altKey && !e.shiftKey) { e.stopImmediatePropagation(); e.preventDefault(); setTimeout(() => this._repeat_last_action(), 0); } - if (e.key === "e" && e.ctrlKey && !e.altKey && !e.shiftKey) { + if (key === "e" && e.ctrlKey && !e.altKey && !e.shiftKey) { e.stopImmediatePropagation(); e.preventDefault(); setTimeout(() => this._flash_fill(), 0); } - if (e.key === "d" && e.ctrlKey && !e.altKey && !e.shiftKey) { + if (key === "d" && e.ctrlKey && !e.altKey && !e.shiftKey) { e.stopImmediatePropagation(); e.preventDefault(); setTimeout(() => this._fill_down(), 0); @@ -580,9 +603,13 @@ frappe.views.ExcelBoard = class ExcelBoard { }, // i18n language: frappe.boot.lang === "ar" || frappe.boot.lang === "he" ? "ar-AR" : undefined, - }); + // Undo / Redo that survives the reload that follows every auto-save. + // (Handsontable's own undo list is erased by each loadData().) + this.undo_manager = new frappe.views.excel.UndoManager({ board: this }); + this.undo_manager.attach(this.hot); + // HOT height:"100%" reads clientHeight at init time — in a flex layout that // value may be 0 before the browser has painted. Force a correct pixel height // after the next paint so HOT's scroll containers initialise properly. @@ -598,9 +625,8 @@ frappe.views.ExcelBoard = class ExcelBoard { // Instead we listen to the master scrollable container (.wtHolder) or // fall back to the window scroll event. const holder = this.$hot_container[0].querySelector(".wtHolder"); - const scroll_target = (holder && holder.scrollHeight > holder.clientHeight) - ? holder - : window; + const scroll_target = + holder && holder.scrollHeight > holder.clientHeight ? holder : window; const on_scroll = () => this._on_scroll_vertical(); scroll_target.addEventListener("scroll", on_scroll, { passive: true }); @@ -649,8 +675,7 @@ frappe.views.ExcelBoard = class ExcelBoard { } const letter = this._col_idx_to_letter(col); const name = frappe.utils.escape_html(col_cfg?.title || letter); - const ct_badge = col_cfg?._is_ct_col - ? `CT` : ""; + const ct_badge = col_cfg?._is_ct_col ? `CT` : ""; return `
${ct_badge}${letter}${name}
`; } @@ -668,6 +693,20 @@ frappe.views.ExcelBoard = class ExcelBoard { return result; } + // A cell the server refused (set by DataManager after a failed save): red + message. + // Called from afterRenderer AFTER _apply_cell_format, which has several early returns + // (for example Long Text cells set their own tooltip) that must not hide the mark. + _mark_failed_cell(TD, row, col) { + if (!this._failed_cells?.size) return; + const rec = this.list_view?.data?.[row]; + const field = this.columns[col]?.data; + const msg = rec && field && this._failed_cells.get(`${rec.name}|${field}`); + if (msg) { + TD.style.backgroundColor = "#ffb3b3"; + TD.title = msg; + } + } + // ── Cell formatting (afterRenderer hook) ────────────────────────────────── /** @@ -687,12 +726,19 @@ frappe.views.ExcelBoard = class ExcelBoard { TD.style.padding = "0"; return; } - if (d && !d._is_new && !d._is_ct_spacer && this.child_table_manager?.has_child_tables()) { + if ( + d && + !d._is_new && + !d._is_ct_spacer && + this.child_table_manager?.has_child_tables() + ) { const is_open = this.child_table_manager?._state?.parent_row_idx === row; TD.innerHTML = `
- ${can_edit - ? ` + ${ + can_edit + ? ` ${__("Editing")} ` - : ` + : ` ${__("View Only")} - `} + ` + } - ${can_edit - ? `
@@ -1075,8 +1143,8 @@ frappe.views.ExcelBoard = class ExcelBoard { `).appendTo(document.body); // ── Inject Frappe editor control ───────────────────────────────────────── - const $area = $overlay.find(".ev-rte-editor-area"); - let _control = null; + const $area = $overlay.find(".ev-rte-editor-area"); + let _control = null; try { _control = frappe.ui.form.make_control({ @@ -1086,11 +1154,17 @@ frappe.views.ExcelBoard = class ExcelBoard { label: "", read_only: can_edit ? 0 : 1, }, - parent: $area[0], + parent: $area[0], render_input: true, }); // Set value after ProseMirror mounts (needs one tick) - setTimeout(() => { try { _control.set_value(html_val); } catch (_) {} }, 60); + setTimeout(() => { + try { + _control.set_value(html_val); + } catch (_) { + // the editor may not be ready yet; the value is set again on the next edit + } + }, 60); } catch (_e) { // Fallback: raw textarea $area.html(`